Compile raylet cython bindings with bazel (#3842)

BUILD.bazel

@@ -2,6 +2,7 @@
 # C/C++ documentation: https://docs.bazel.build/versions/master/be/c-cpp.html
 
 load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library")
+load("@//bazel:cython_library.bzl", "pyx_library")
 
 cc_binary(
     name = "raylet",
@@ -45,7 +46,7 @@ cc_library(
         ":ray_common",
         ":ray_util",
         "@boost//:asio",
-        "@plasma",
+        "@plasma//:plasma_client",
         "@com_google_googletest//:gtest",
     ],
 )
@@ -124,7 +125,7 @@ cc_library(
         ":ray_common",
         ":ray_util",
         "@boost//:asio",
-        "@plasma"
+        "@plasma//:plasma_client"
     ],
 )
 
@@ -165,7 +166,7 @@ cc_library(
         "src",
     ],
     deps = [
-      "@plasma",
+      "@plasma//:plasma_client",
       ":sha256"
     ]
 )
@@ -304,3 +305,14 @@ flatbuffer_cc_library(
     flatc_args = FLATC_ARGS,
     out_prefix = "src/ray/object_manager/format/",
 )
+
+pyx_library(
+    name = "_raylet",
+    srcs = glob([
+        "python/ray/__init__.py",
+        "python/ray/_raylet.pyx",
+        "python/ray/includes/*.pxd",
+        "python/ray/includes/*.pxi"
+    ]),
+    deps = ["//:raylet_lib"]
+)

WORKSPACE

@@ -25,5 +25,15 @@ new_git_repository(
     name = "plasma",
     build_file = "@//bazel:BUILD.plasma",
     remote = "https://github.com/ray-project/arrow",
-    commit = "1e4f867eb1dc31107331ab1defdffb94467f31dc",
+    commit = "f5d1be2fed69899aea636bd074aaeaa4149acc79",
 )
+
+new_git_repository(
+    name="cython",
+    build_file="@//bazel:BUILD.cython",
+    remote = "https://github.com/cython/cython",
+    commit = "49414dbc7ddc2ca2979d6dbe1e44714b10d72e7e",
+)
+
+load("@//bazel:python_configure.bzl", "python_configure")
+python_configure(name="local_config_python")

bazel/BUILD.cython

@@ -0,0 +1,30 @@
+# Adapted from grpc/third_party/cython.BUILD
+
+# Adapted with modifications from tensorflow/third_party/cython.BUILD
+
+py_library(
+    name="cython_lib",
+    srcs=glob(
+        ["Cython/**/*.py"],
+        exclude=[
+            "**/Tests/*.py",
+        ],
+    ) + ["cython.py"],
+    data=glob([
+        "Cython/**/*.pyx",
+        "Cython/Utility/*.*",
+        "Cython/Includes/**/*.pxd",
+    ]),
+    srcs_version="PY2AND3",
+    visibility=["//visibility:public"],
+)
+
+# May not be named "cython", since that conflicts with Cython/ on OSX
+py_binary(
+    name="cython_binary",
+    srcs=["cython.py"],
+    main="cython.py",
+    srcs_version="PY2AND3",
+    visibility=["//visibility:public"],
+    deps=["cython_lib"],
+)

bazel/BUILD.plasma

@@ -1,28 +1,70 @@
 load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library")
 
 cc_library(
-    name = "plasma",
+    name = "arrow",
+    hdrs = [
+        "cpp/src/arrow/buffer.h",
+        "cpp/src/arrow/memory_pool.h",
+        "cpp/src/arrow/status.h",
+        "cpp/src/arrow/io/interfaces.h",
+        "cpp/src/arrow/util/bit-util.h",
+        "cpp/src/arrow/util/io-util.h",
+        "cpp/src/arrow/util/logging.h",
+        "cpp/src/arrow/util/macros.h",
+        "cpp/src/arrow/util/string_builder.h",
+        "cpp/src/arrow/util/string_view.h",
+        "cpp/src/arrow/util/type_traits.h",
+        "cpp/src/arrow/util/thread-pool.h",
+        "cpp/src/arrow/util/visibility.h",
+        "cpp/src/arrow/util/windows_compatibility.h",
+        "cpp/src/arrow/vendored/string_view.hpp",
+        "cpp/src/arrow/vendored/xxhash/xxhash.h",
+        "cpp/src/arrow/vendored/xxhash/xxhash.c",
+    ],
+    srcs = [
+        "cpp/src/arrow/buffer.cc",
+        "cpp/src/arrow/memory_pool.cc",
+        "cpp/src/arrow/status.cc",
+        "cpp/src/arrow/util/logging.cc",
+        "cpp/src/arrow/util/thread-pool.cc",
+        "cpp/src/arrow/util/io-util.cc",
+    ],
+    strip_include_prefix = "cpp/src",
+)
+
+cc_library(
+    name = "plasma_client",
     visibility = ["//visibility:public"],
     hdrs = [
         "cpp/src/plasma/client.h",
         "cpp/src/plasma/common.h",
         "cpp/src/plasma/compat.h",
-        "cpp/src/plasma/events.h",
-        "cpp/src/arrow/buffer.h",
-        "cpp/src/arrow/memory_pool.h",
-        "cpp/src/arrow/status.h",
-        "cpp/src/arrow/util/macros.h",
-        "cpp/src/arrow/util/visibility.h",
-        "cpp/src/arrow/util/string_builder.h",
-        # private headers
-        "cpp/src/plasma/common_generated.h",
-        "cpp/src/plasma/plasma_generated.h",
-        "cpp/src/plasma/eviction_policy.h",
         "cpp/src/plasma/fling.h",
         "cpp/src/plasma/io.h",
         "cpp/src/plasma/malloc.h",
         "cpp/src/plasma/plasma.h",
         "cpp/src/plasma/protocol.h",
+        "cpp/src/plasma/common_generated.h",
+        "cpp/src/plasma/plasma_generated.h",
+    ],
+    srcs = [
+        "cpp/src/plasma/client.cc",
+        "cpp/src/plasma/common.cc",
+        "cpp/src/plasma/fling.cc",
+        "cpp/src/plasma/io.cc",
+        "cpp/src/plasma/malloc.cc",
+        "cpp/src/plasma/plasma.cc",
+        "cpp/src/plasma/protocol.cc",
+    ],
+    deps = [":common_fbs", ":plasma_fbs", ":arrow"],
+    strip_include_prefix = "cpp/src",
+)
+
+cc_library(
+    name = "plasma_lib",
+    hdrs = [
+        "cpp/src/plasma/events.h",
+        "cpp/src/plasma/eviction_policy.h",
         "cpp/src/plasma/store.h",
         "cpp/src/plasma/thirdparty/dlmalloc.c",
         "cpp/src/plasma/thirdparty/ae/ae.h",
@@ -32,32 +74,27 @@ cc_library(
         "cpp/src/plasma/thirdparty/ae/ae_select.c",
         "cpp/src/plasma/thirdparty/ae/config.h",
         "cpp/src/plasma/thirdparty/ae/zmalloc.h",
-        "cpp/src/arrow/io/interfaces.h",
-        "cpp/src/arrow/util/bit-util.h",
-        "cpp/src/arrow/util/io-util.h",
-        "cpp/src/arrow/util/logging.h",
-        "cpp/src/arrow/util/string_view.h",
-        "cpp/src/arrow/util/type_traits.h",
-        "cpp/src/arrow/vendored/string_view.hpp",
-        "cpp/src/arrow/util/thread-pool.h",
-        "cpp/src/arrow/vendored/xxhash/xxhash.h",
-        "cpp/src/arrow/vendored/xxhash/xxhash.c",
-        "cpp/src/arrow/util/windows_compatibility.h",
     ],
-    srcs = glob([
-      "cpp/src/plasma/*.cc",
-      "cpp/src/plasma/thirdparty/ae/ae.c",
-      "cpp/src/arrow/buffer.cc",
-      "cpp/src/arrow/memory_pool.cc",
-      "cpp/src/arrow/status.cc",
-      "cpp/src/arrow/util/logging.cc",
-      "cpp/src/arrow/util/thread-pool.cc",
-      "cpp/src/arrow/util/io-util.cc",
-    ]),
-    deps = [":common_fbs", ":plasma_fbs"],
+    srcs = [
+        "cpp/src/plasma/dlmalloc.cc",
+        "cpp/src/plasma/eviction_policy.cc",
+        "cpp/src/plasma/events.cc",
+    ],
+    deps = [":plasma_client"],
     strip_include_prefix = "cpp/src",
 )
 
+
+cc_binary(
+    name = "plasma_server",
+    visibility = ["//visibility:public"],
+    srcs = glob([
+      "cpp/src/plasma/store.cc",
+      "cpp/src/plasma/thirdparty/ae/ae.c",
+    ], exclude = ["cpp/src/plasma/dlmalloc.cc", "cpp/src/plasma/events.cc"]),
+    deps = [":plasma_lib"],
+)
+
 FLATC_ARGS = [
     "--gen-object-api",
     "--gen-mutable",

bazel/BUILD.tpl

@@ -0,0 +1,36 @@
+# Adapted from grpc/third_party/py/BUILD.tpl
+
+# Adapted with modifications from tensorflow/third_party/py/
+
+package(default_visibility=["//visibility:public"])
+
+# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
+# See https://docs.python.org/3/extending/windows.html
+cc_import(
+    name="python_lib",
+    interface_library=select({
+        ":windows": ":python_import_lib",
+        # A placeholder for Unix platforms which makes --no_build happy.
+        "//conditions:default": "not-existing.lib",
+    }),
+    system_provided=1,
+)
+
+cc_library(
+    name="python_headers",
+    hdrs=[":python_include"],
+    deps=select({
+        ":windows": [":python_lib"],
+        "//conditions:default": [],
+    }),
+    includes=["python_include"],
+)
+
+config_setting(
+    name="windows",
+    values={"cpu": "x64_windows"},
+    visibility=["//visibility:public"],
+)
+
+%{PYTHON_INCLUDE_GENRULE}
+%{PYTHON_IMPORT_LIB_GENRULE}

bazel/cython_library.bzl

@@ -0,0 +1,72 @@
+"""Custom rules for gRPC Python"""
+
+# Adapted from grpc/bazel/cython_library.bzl
+
+# Adapted with modifications from
+# tensorflow/tensorflow/core/platform/default/build_config.bzl
+# Native Bazel rules don't exist yet to compile Cython code, but rules have
+# been written at cython/cython and tensorflow/tensorflow. We branch from
+# Tensorflow's version as it is more actively maintained and works for gRPC
+# Python's needs.
+def pyx_library(name, deps=[], py_deps=[], srcs=[], **kwargs):
+    """Compiles a group of .pyx / .pxd / .py files.
+    First runs Cython to create .cpp files for each input .pyx or .py + .pxd
+    pair. Then builds a shared object for each, passing "deps" to each cc_binary
+    rule (includes Python headers by default). Finally, creates a py_library rule
+    with the shared objects and any pure Python "srcs", with py_deps as its
+    dependencies; the shared objects can be imported like normal Python files.
+    Args:
+        name: Name for the rule.
+        deps: C/C++ dependencies of the Cython (e.g. Numpy headers).
+        py_deps: Pure Python dependencies of the final library.
+        srcs: .py, .pyx, or .pxd files to either compile or pass through.
+        **kwargs: Extra keyword arguments passed to the py_library.
+    """
+    # First filter out files that should be run compiled vs. passed through.
+    py_srcs = []
+    pyx_srcs = []
+    pxd_srcs = []
+    for src in srcs:
+        if src.endswith(".pyx") or (src.endswith(".py") and
+                                    src[:-3] + ".pxd" in srcs):
+            pyx_srcs.append(src)
+        elif src.endswith(".py"):
+            py_srcs.append(src)
+        else:
+            pxd_srcs.append(src)
+        if src.endswith("__init__.py"):
+            pxd_srcs.append(src)
+
+    # Invoke cython to produce the shared object libraries.
+    for filename in pyx_srcs:
+        native.genrule(
+            name=filename + "_cython_translation",
+            srcs=[filename],
+            outs=[filename.split(".")[0] + ".cpp"],
+            # Optionally use PYTHON_BIN_PATH on Linux platforms so that python 3
+            # works. Windows has issues with cython_binary so skip PYTHON_BIN_PATH.
+            cmd=
+            "PYTHONHASHSEED=0 $(location @cython//:cython_binary) --cplus $(SRCS) --output-file $(OUTS)",
+            tools=["@cython//:cython_binary"] + pxd_srcs,
+        )
+
+    shared_objects = []
+    for src in pyx_srcs:
+        stem = src.split(".")[0]
+        shared_object_name = stem + ".so"
+        native.cc_binary(
+            name=shared_object_name,
+            srcs=[stem + ".cpp"],
+            deps=deps + ["@local_config_python//:python_headers"],
+            linkshared=1,
+        )
+        shared_objects.append(shared_object_name)
+
+    # Now create a py_library with these shared objects as data.
+    native.py_library(
+        name=name,
+        srcs=py_srcs,
+        deps=py_deps,
+        srcs_version="PY2AND3",
+        data=shared_objects,
+        **kwargs)

bazel/python_configure.bzl

@@ -0,0 +1,299 @@
+# Adapted with a minor modification from grpc/third_party/py/python_configure.bzl:
+# Changed "//third_party/py:%s.tpl" -> "@//bazel:%s.tpl"
+
+# Adapted with modifications from tensorflow/third_party/py/
+
+"""Repository rule for Python autoconfiguration.
+`python_configure` depends on the following environment variables:
+  * `PYTHON_BIN_PATH`: location of python binary.
+  * `PYTHON_LIB_PATH`: Location of python libraries.
+"""
+
+_BAZEL_SH = "BAZEL_SH"
+_PYTHON_BIN_PATH = "PYTHON_BIN_PATH"
+_PYTHON_LIB_PATH = "PYTHON_LIB_PATH"
+_PYTHON_CONFIG_REPO = "PYTHON_CONFIG_REPO"
+
+
+def _tpl(repository_ctx, tpl, substitutions={}, out=None):
+    if not out:
+        out = tpl
+    repository_ctx.template(out, Label("@//bazel:%s.tpl" % tpl),
+                            substitutions)
+
+
+def _fail(msg):
+    """Output failure message when auto configuration fails."""
+    red = "\033[0;31m"
+    no_color = "\033[0m"
+    fail("%sPython Configuration Error:%s %s\n" % (red, no_color, msg))
+
+
+def _is_windows(repository_ctx):
+    """Returns true if the host operating system is windows."""
+    os_name = repository_ctx.os.name.lower()
+    return os_name.find("windows") != -1
+
+
+def _execute(repository_ctx,
+             cmdline,
+             error_msg=None,
+             error_details=None,
+             empty_stdout_fine=False):
+    """Executes an arbitrary shell command.
+    Args:
+        repository_ctx: the repository_ctx object
+        cmdline: list of strings, the command to execute
+        error_msg: string, a summary of the error if the command fails
+        error_details: string, details about the error or steps to fix it
+        empty_stdout_fine: bool, if True, an empty stdout result is fine, otherwise
+        it's an error
+    Return:
+        the result of repository_ctx.execute(cmdline)
+  """
+    result = repository_ctx.execute(cmdline)
+    if result.stderr or not (empty_stdout_fine or result.stdout):
+        _fail("\n".join([
+            error_msg.strip() if error_msg else "Repository command failed",
+            result.stderr.strip(), error_details if error_details else ""
+        ]))
+    else:
+        return result
+
+
+def _read_dir(repository_ctx, src_dir):
+    """Returns a string with all files in a directory.
+  Finds all files inside a directory, traversing subfolders and following
+  symlinks. The returned string contains the full path of all files
+  separated by line breaks.
+  """
+    if _is_windows(repository_ctx):
+        src_dir = src_dir.replace("/", "\\")
+        find_result = _execute(
+            repository_ctx,
+            ["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"],
+            empty_stdout_fine=True)
+        # src_files will be used in genrule.outs where the paths must
+        # use forward slashes.
+        return find_result.stdout.replace("\\", "/")
+    else:
+        find_result = _execute(
+            repository_ctx, ["find", src_dir, "-follow", "-type", "f"],
+            empty_stdout_fine=True)
+        return find_result.stdout
+
+
+def _genrule(src_dir, genrule_name, command, outs):
+    """Returns a string with a genrule.
+  Genrule executes the given command and produces the given outputs.
+  """
+    return ('genrule(\n' + '    name = "' + genrule_name + '",\n' +
+            '    outs = [\n' + outs + '\n    ],\n' + '    cmd = """\n' +
+            command + '\n   """,\n' + ')\n')
+
+
+def _normalize_path(path):
+    """Returns a path with '/' and remove the trailing slash."""
+    path = path.replace("\\", "/")
+    if path[-1] == "/":
+        path = path[:-1]
+    return path
+
+
+def _symlink_genrule_for_dir(repository_ctx,
+                             src_dir,
+                             dest_dir,
+                             genrule_name,
+                             src_files=[],
+                             dest_files=[]):
+    """Returns a genrule to symlink(or copy if on Windows) a set of files.
+  If src_dir is passed, files will be read from the given directory; otherwise
+  we assume files are in src_files and dest_files
+  """
+    if src_dir != None:
+        src_dir = _normalize_path(src_dir)
+        dest_dir = _normalize_path(dest_dir)
+        files = '\n'.join(
+            sorted(_read_dir(repository_ctx, src_dir).splitlines()))
+        # Create a list with the src_dir stripped to use for outputs.
+        dest_files = files.replace(src_dir, '').splitlines()
+        src_files = files.splitlines()
+    command = []
+    outs = []
+    for i in range(len(dest_files)):
+        if dest_files[i] != "":
+            # If we have only one file to link we do not want to use the dest_dir, as
+            # $(@D) will include the full path to the file.
+            dest = '$(@D)/' + dest_dir + dest_files[i] if len(
+                dest_files) != 1 else '$(@D)/' + dest_files[i]
+            # On Windows, symlink is not supported, so we just copy all the files.
+            cmd = 'cp -f' if _is_windows(repository_ctx) else 'ln -s'
+            command.append(cmd + ' "%s" "%s"' % (src_files[i], dest))
+            outs.append('        "' + dest_dir + dest_files[i] + '",')
+    return _genrule(src_dir, genrule_name, " && ".join(command),
+                    "\n".join(outs))
+
+
+def _get_python_bin(repository_ctx):
+    """Gets the python bin path."""
+    python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH)
+    if python_bin != None:
+        return python_bin
+    python_bin_path = repository_ctx.which("python")
+    if python_bin_path != None:
+        return str(python_bin_path)
+    _fail("Cannot find python in PATH, please make sure " +
+          "python is installed and add its directory in PATH, or --define " +
+          "%s='/something/else'.\nPATH=%s" %
+          (_PYTHON_BIN_PATH, repository_ctx.os.environ.get("PATH", "")))
+
+
+def _get_bash_bin(repository_ctx):
+    """Gets the bash bin path."""
+    bash_bin = repository_ctx.os.environ.get(_BAZEL_SH)
+    if bash_bin != None:
+        return bash_bin
+    else:
+        bash_bin_path = repository_ctx.which("bash")
+        if bash_bin_path != None:
+            return str(bash_bin_path)
+        else:
+            _fail(
+                "Cannot find bash in PATH, please make sure " +
+                "bash is installed and add its directory in PATH, or --define "
+                + "%s='/path/to/bash'.\nPATH=%s" %
+                (_BAZEL_SH, repository_ctx.os.environ.get("PATH", "")))
+
+
+def _get_python_lib(repository_ctx, python_bin):
+    """Gets the python lib path."""
+    python_lib = repository_ctx.os.environ.get(_PYTHON_LIB_PATH)
+    if python_lib != None:
+        return python_lib
+    print_lib = (
+        "<<END\n" + "from __future__ import print_function\n" +
+        "import site\n" + "import os\n" + "\n" + "try:\n" +
+        "  input = raw_input\n" + "except NameError:\n" + "  pass\n" + "\n" +
+        "python_paths = []\n" + "if os.getenv('PYTHONPATH') is not None:\n" +
+        "  python_paths = os.getenv('PYTHONPATH').split(':')\n" + "try:\n" +
+        "  library_paths = site.getsitepackages()\n" +
+        "except AttributeError:\n" +
+        " from distutils.sysconfig import get_python_lib\n" +
+        " library_paths = [get_python_lib()]\n" +
+        "all_paths = set(python_paths + library_paths)\n" + "paths = []\n" +
+        "for path in all_paths:\n" + "  if os.path.isdir(path):\n" +
+        "    paths.append(path)\n" + "if len(paths) >=1:\n" +
+        "  print(paths[0])\n" + "END")
+    cmd = '%s - %s' % (python_bin, print_lib)
+    result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd])
+    return result.stdout.strip('\n')
+
+
+def _check_python_lib(repository_ctx, python_lib):
+    """Checks the python lib path."""
+    cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib)
+    result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd])
+    if result.return_code == 1:
+        _fail("Invalid python library path: %s" % python_lib)
+
+
+def _check_python_bin(repository_ctx, python_bin):
+    """Checks the python bin path."""
+    cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin)
+    result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd])
+    if result.return_code == 1:
+        _fail("--define %s='%s' is not executable. Is it the python binary?" %
+              (_PYTHON_BIN_PATH, python_bin))
+
+
+def _get_python_include(repository_ctx, python_bin):
+    """Gets the python include path."""
+    result = _execute(
+        repository_ctx, [
+            python_bin, "-c", 'from __future__ import print_function;' +
+            'from distutils import sysconfig;' +
+            'print(sysconfig.get_python_inc())'
+        ],
+        error_msg="Problem getting python include path.",
+        error_details=(
+            "Is the Python binary path set up right? " + "(See ./configure or "
+            + _PYTHON_BIN_PATH + ".) " + "Is distutils installed?"))
+    return result.stdout.splitlines()[0]
+
+
+def _get_python_import_lib_name(repository_ctx, python_bin):
+    """Get Python import library name (pythonXY.lib) on Windows."""
+    result = _execute(
+        repository_ctx, [
+            python_bin, "-c",
+            'import sys;' + 'print("python" + str(sys.version_info[0]) + ' +
+            '      str(sys.version_info[1]) + ".lib")'
+        ],
+        error_msg="Problem getting python import library.",
+        error_details=("Is the Python binary path set up right? " +
+                       "(See ./configure or " + _PYTHON_BIN_PATH + ".) "))
+    return result.stdout.splitlines()[0]
+
+
+def _create_local_python_repository(repository_ctx):
+    """Creates the repository containing files set up to build with Python."""
+    python_bin = _get_python_bin(repository_ctx)
+    _check_python_bin(repository_ctx, python_bin)
+    python_lib = _get_python_lib(repository_ctx, python_bin)
+    _check_python_lib(repository_ctx, python_lib)
+    python_include = _get_python_include(repository_ctx, python_bin)
+    python_include_rule = _symlink_genrule_for_dir(
+        repository_ctx, python_include, 'python_include', 'python_include')
+    python_import_lib_genrule = ""
+    # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
+    # See https://docs.python.org/3/extending/windows.html
+    if _is_windows(repository_ctx):
+        python_include = _normalize_path(python_include)
+        python_import_lib_name = _get_python_import_lib_name(
+            repository_ctx, python_bin)
+        python_import_lib_src = python_include.rsplit(
+            '/', 1)[0] + "/libs/" + python_import_lib_name
+        python_import_lib_genrule = _symlink_genrule_for_dir(
+            repository_ctx, None, '', 'python_import_lib',
+            [python_import_lib_src], [python_import_lib_name])
+    _tpl(
+        repository_ctx, "BUILD", {
+            "%{PYTHON_INCLUDE_GENRULE}": python_include_rule,
+            "%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule,
+        })
+
+
+def _create_remote_python_repository(repository_ctx, remote_config_repo):
+    """Creates pointers to a remotely configured repo set up to build with Python.
+  """
+    _tpl(repository_ctx, "remote.BUILD", {
+        "%{REMOTE_PYTHON_REPO}": remote_config_repo,
+    }, "BUILD")
+
+
+def _python_autoconf_impl(repository_ctx):
+    """Implementation of the python_autoconf repository rule."""
+    if _PYTHON_CONFIG_REPO in repository_ctx.os.environ:
+        _create_remote_python_repository(
+            repository_ctx, repository_ctx.os.environ[_PYTHON_CONFIG_REPO])
+    else:
+        _create_local_python_repository(repository_ctx)
+
+
+python_configure = repository_rule(
+    implementation=_python_autoconf_impl,
+    environ=[
+        _BAZEL_SH,
+        _PYTHON_BIN_PATH,
+        _PYTHON_LIB_PATH,
+        _PYTHON_CONFIG_REPO,
+    ],
+)
+"""Detects and configures the local Python.
+Add the following to your WORKSPACE FILE:
+```python
+python_configure(name = "local_config_python")
+```
+Args:
+  name: A unique name for this workspace rule.
+"""

python/ray/_raylet.pyx

@@ -11,6 +11,13 @@ from libcpp.utility cimport pair
 from libcpp.unordered_map cimport unordered_map
 from libcpp.vector cimport vector as c_vector
 
+from cython.operator import dereference, postincrement
+cimport cpython
+
+include "includes/unique_ids.pxi"
+include "includes/ray_config.pxi"
+include "includes/task.pxi"
+
 from ray.includes.common cimport (
     CUniqueID, CTaskID, CObjectID, CFunctionID, CActorClassID, CActorID,
     CActorHandleID, CWorkerID, CDriverID, CConfigID, CClientID,
@@ -22,13 +29,6 @@ from ray.includes.task cimport CTaskSpecification
 from ray.includes.ray_config cimport RayConfig
 from ray.utils import decode
 
-from cython.operator import dereference, postincrement
-cimport cpython
-
-include "includes/unique_ids.pxi"
-include "includes/ray_config.pxi"
-include "includes/task.pxi"
-
 
 if cpython.PY_MAJOR_VERSION >= 3:
     import pickle

src/ray/gcs/client.cc

@@ -158,12 +158,6 @@ AsyncGcsClient::AsyncGcsClient(const std::string &address, int port,
 AsyncGcsClient::AsyncGcsClient(const std::string &address, int port, bool is_test_client)
     : AsyncGcsClient(address, port, ClientID::from_random(), is_test_client) {}
 
-Status Attach(plasma::EventLoop &event_loop) {
-  // TODO(pcm): Implement this via
-  // context()->AttachToEventLoop(event loop)
-  return Status::OK();
-}
-
 Status AsyncGcsClient::Attach(boost::asio::io_service &io_service) {
   // Take care of sharding contexts.
   RAY_CHECK(shard_asio_async_clients_.empty()) << "Attach shall be called only once";

src/ray/gcs/client.h

@@ -4,7 +4,6 @@
 #include <map>
 #include <string>
 
-#include "plasma/events.h"
 #include "ray/gcs/asio.h"
 #include "ray/gcs/tables.h"
 #include "ray/id.h"
@@ -41,9 +40,6 @@ class RAY_EXPORT AsyncGcsClient {
   AsyncGcsClient(const std::string &address, int port, const std::string &password);
   AsyncGcsClient(const std::string &address, int port, bool is_test_client);
 
-  /// Attach this client to a plasma event loop. Note that only
-  /// one event loop should be attached at a time.
-  Status Attach(plasma::EventLoop &event_loop);
   /// Attach this client to an asio event loop. Note that only
   /// one event loop should be attached at a time.
   Status Attach(boost::asio::io_service &io_service);

src/ray/object_manager/object_buffer_pool.h

@@ -11,7 +11,6 @@
 #include <boost/bind.hpp>
 
 #include "plasma/client.h"
-#include "plasma/events.h"
 
 #include "ray/id.h"
 #include "ray/status.h"

src/ray/object_manager/object_manager.h

@@ -14,7 +14,6 @@
 #include <boost/bind.hpp>
 
 #include "plasma/client.h"
-#include "plasma/events.h"
 
 #include "ray/common/client_connection.h"
 #include "ray/id.h"

src/ray/object_manager/object_store_notification_manager.h

@@ -10,7 +10,6 @@
 #include <boost/bind.hpp>
 
 #include "plasma/client.h"
-#include "plasma/events.h"
 
 #include "ray/id.h"
 #include "ray/status.h"