Add script to build manylinux1 .whl files (#600)

* Add manylinux setup

* Switch to cp27mu

* python/MANIFEST.in

* Fix MANIFEST.in

* Add build-wheel-manylinux1.sh

* Update readme

* Install correct version of numpy

* Fix typo in README-manylinux1.md

* Don't install cmake

* Remove commented line from setup.py

* Delete unused manylinux1.sh

* Run setup.py bdist_wheel twice

* Don't use package_data and MANIFEST.in.

* Small aesthetic change.

* Trigger build_ext in setup.py.

* Remove nonexistent file from MANIFEST.in.

* Manually copy files in MANIFEST.in to where Python expects them in order to prevent setup.py from having to be run twice.

* Only run setup.py once when building wheels.

* Aesthetic change to readme.

* Copy generated flatbuffer Python files in build_ext.

* Fix permission denied error by making sure to preserve executableness when copying files.

* Remove unnecessary argument to setup.py.

* Remove MANIFEST.in and move files to include into list in setup.py.

* Fix numpy version when building wheels and replace rm with git clean.

python/setup.py

@@ -2,39 +2,67 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import os
+import shutil
 import subprocess
 
-from setuptools import setup, find_packages
-import setuptools.command.install as _install
+from setuptools import setup, find_packages, Distribution
+import setuptools.command.build_ext as _build_ext
 
 
-class install(_install.install):
+class build_ext(_build_ext.build_ext):
   def run(self):
     subprocess.check_call(["../build.sh"])
-    # Calling _install.install.run(self) does not fetch required packages and
-    # instead performs an old-style install. See command/install.py in
-    # setuptools. So, calling do_egg_install() manually here.
-    self.do_egg_install()
+    # Ideally, we could include these files by putting them in a MANIFEST.in or
+    # using the package_data argument to setup, but the MANIFEST.in gets
+    # applied at the very beginning when setup.py runs before these files have
+    # been created, so we have to move the files manually.
+    for filename in files_to_include:
+      self.move_file(filename)
+    # Copy over the autogenerated flatbuffer Python bindings.
+    generated_python_directory = "ray/core/generated"
+    for filename in os.listdir(generated_python_directory):
+      if filename[-3:] == ".py":
+        self.move_file(os.path.join(generated_python_directory, filename))
+
+  def move_file(self, filename):
+    # TODO(rkn): This feels very brittle. It may not handle all cases. See
+    # https://github.com/apache/arrow/blob/master/python/setup.py for an
+    # example.
+    source = filename
+    destination = os.path.join(self.build_lib, filename)
+    # Create the target directory if it doesn't already exist.
+    parent_directory = os.path.dirname(destination)
+    if not os.path.exists(parent_directory):
+      os.makedirs(parent_directory)
+    print("Copying {} to {}.".format(source, destination))
+    shutil.copy(source, destination)
 
 
-package_data = {
-    "ray": ["core/src/common/thirdparty/redis/src/redis-server",
-            "core/src/common/redis_module/libray_redis_module.so",
-            "core/src/plasma/plasma_store",
-            "core/src/plasma/plasma_manager",
-            "core/src/plasma/libplasma.so",
-            "core/src/local_scheduler/local_scheduler",
-            "core/src/local_scheduler/liblocal_scheduler_library.so",
-            "core/src/numbuf/libarrow.so",
-            "core/src/numbuf/libnumbuf.so",
-            "core/src/global_scheduler/global_scheduler"]
-}
+files_to_include = [
+    "ray/core/src/common/thirdparty/redis/src/redis-server",
+    "ray/core/src/common/redis_module/libray_redis_module.so",
+    "ray/core/src/plasma/plasma_store",
+    "ray/core/src/plasma/plasma_manager",
+    "ray/core/src/plasma/libplasma.so",
+    "ray/core/src/local_scheduler/local_scheduler",
+    "ray/core/src/local_scheduler/liblocal_scheduler_library.so",
+    "ray/core/src/numbuf/libnumbuf.so",
+    "ray/core/src/global_scheduler/global_scheduler"
+]
+
+
+class BinaryDistribution(Distribution):
+  def has_ext_modules(self):
+    return True
+
 
 setup(name="ray",
       version="0.1.0",
       packages=find_packages(),
-      package_data=package_data,
-      cmdclass={"install": install},
+      cmdclass={"build_ext": build_ext},
+      # The BinaryDistribution argument triggers build_ext.
+      distclass=BinaryDistribution,
       install_requires=["numpy",
                         "funcsigs",
                         "colorama",