diff --git a/python/README-manylinux1.md b/python/README-manylinux1.md new file mode 100644 index 000000000..f8055d19d --- /dev/null +++ b/python/README-manylinux1.md @@ -0,0 +1,15 @@ +# Building manylinux1 wheels + +To cause everything to be rebuilt, this script will delete ALL changes to the +repository, including both changes to tracked files, and ANY untracked files. + +It will also cause all files inside the repository to be owned by root, and +produce .whl files owned by root. + +Inside the root directory (i.e. one level above this python directory), run + +``` +docker run --rm -w /ray -v `pwd`:/ray -ti quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-1024 /ray/python/build-wheel-manylinux1.sh +``` + +The wheel files will be placed in the .whl directory. diff --git a/python/build-wheel-manylinux1.sh b/python/build-wheel-manylinux1.sh new file mode 100755 index 000000000..5a11a8fa7 --- /dev/null +++ b/python/build-wheel-manylinux1.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +cat << EOF > "/usr/bin/nproc" +#!/bin/bash +echo 1 +EOF +chmod +x /usr/bin/nproc + +mkdir .whl +for PYTHON in cp27-cp27mu cp33-cp33m cp34-cp34m cp35-cp35m cp36-cp36m; do + # The -f flag is passed twice to also run git clean in the arrow subdirectory. + # The -d flag removes directories. The -x flag ignores the .gitignore file, + # and the -e flag ensures that we don't remove the .whl directory. + git clean -f -f -x -d -e .whl + pushd python + # Fix the numpy version because this will be the oldest numpy version we can + # support. + /opt/python/${PYTHON}/bin/pip install numpy==1.10.4 + PATH=/opt/python/${PYTHON}/bin:$PATH /opt/python/${PYTHON}/bin/python setup.py bdist_wheel + # In the future, run auditwheel here. + mv dist/*.whl ../.whl/ + popd +done diff --git a/python/setup.py b/python/setup.py index 6d1250e28..f7b520db5 100644 --- a/python/setup.py +++ b/python/setup.py @@ -2,39 +2,67 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import os +import shutil import subprocess -from setuptools import setup, find_packages -import setuptools.command.install as _install +from setuptools import setup, find_packages, Distribution +import setuptools.command.build_ext as _build_ext -class install(_install.install): +class build_ext(_build_ext.build_ext): def run(self): subprocess.check_call(["../build.sh"]) - # Calling _install.install.run(self) does not fetch required packages and - # instead performs an old-style install. See command/install.py in - # setuptools. So, calling do_egg_install() manually here. - self.do_egg_install() + # Ideally, we could include these files by putting them in a MANIFEST.in or + # using the package_data argument to setup, but the MANIFEST.in gets + # applied at the very beginning when setup.py runs before these files have + # been created, so we have to move the files manually. + for filename in files_to_include: + self.move_file(filename) + # Copy over the autogenerated flatbuffer Python bindings. + generated_python_directory = "ray/core/generated" + for filename in os.listdir(generated_python_directory): + if filename[-3:] == ".py": + self.move_file(os.path.join(generated_python_directory, filename)) + + def move_file(self, filename): + # TODO(rkn): This feels very brittle. It may not handle all cases. See + # https://github.com/apache/arrow/blob/master/python/setup.py for an + # example. + source = filename + destination = os.path.join(self.build_lib, filename) + # Create the target directory if it doesn't already exist. + parent_directory = os.path.dirname(destination) + if not os.path.exists(parent_directory): + os.makedirs(parent_directory) + print("Copying {} to {}.".format(source, destination)) + shutil.copy(source, destination) -package_data = { - "ray": ["core/src/common/thirdparty/redis/src/redis-server", - "core/src/common/redis_module/libray_redis_module.so", - "core/src/plasma/plasma_store", - "core/src/plasma/plasma_manager", - "core/src/plasma/libplasma.so", - "core/src/local_scheduler/local_scheduler", - "core/src/local_scheduler/liblocal_scheduler_library.so", - "core/src/numbuf/libarrow.so", - "core/src/numbuf/libnumbuf.so", - "core/src/global_scheduler/global_scheduler"] -} +files_to_include = [ + "ray/core/src/common/thirdparty/redis/src/redis-server", + "ray/core/src/common/redis_module/libray_redis_module.so", + "ray/core/src/plasma/plasma_store", + "ray/core/src/plasma/plasma_manager", + "ray/core/src/plasma/libplasma.so", + "ray/core/src/local_scheduler/local_scheduler", + "ray/core/src/local_scheduler/liblocal_scheduler_library.so", + "ray/core/src/numbuf/libnumbuf.so", + "ray/core/src/global_scheduler/global_scheduler" +] + + +class BinaryDistribution(Distribution): + def has_ext_modules(self): + return True + setup(name="ray", version="0.1.0", packages=find_packages(), - package_data=package_data, - cmdclass={"install": install}, + cmdclass={"build_ext": build_ext}, + # The BinaryDistribution argument triggers build_ext. + distclass=BinaryDistribution, install_requires=["numpy", "funcsigs", "colorama",