From 20162ce1598afad122dee51cf514475a004ad2af Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Fri, 25 Jan 2019 00:57:31 -0800 Subject: [PATCH] Compile raylet cython bindings with bazel (#3842) --- BUILD.bazel | 18 +- WORKSPACE | 12 +- bazel/BUILD.cython | 30 ++ bazel/BUILD.plasma | 105 ++++-- bazel/BUILD.tpl | 36 +++ bazel/cython_library.bzl | 72 +++++ bazel/python_configure.bzl | 299 ++++++++++++++++++ python/ray/_raylet.pyx | 14 +- src/ray/gcs/client.cc | 6 - src/ray/gcs/client.h | 4 - src/ray/object_manager/object_buffer_pool.h | 1 - src/ray/object_manager/object_manager.h | 1 - .../object_store_notification_manager.h | 1 - 13 files changed, 541 insertions(+), 58 deletions(-) create mode 100644 bazel/BUILD.cython create mode 100644 bazel/BUILD.tpl create mode 100644 bazel/cython_library.bzl create mode 100644 bazel/python_configure.bzl diff --git a/BUILD.bazel b/BUILD.bazel index 2913cdd3c..afb957b5e 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -2,6 +2,7 @@ # C/C++ documentation: https://docs.bazel.build/versions/master/be/c-cpp.html load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library") +load("@//bazel:cython_library.bzl", "pyx_library") cc_binary( name = "raylet", @@ -45,7 +46,7 @@ cc_library( ":ray_common", ":ray_util", "@boost//:asio", - "@plasma", + "@plasma//:plasma_client", "@com_google_googletest//:gtest", ], ) @@ -124,7 +125,7 @@ cc_library( ":ray_common", ":ray_util", "@boost//:asio", - "@plasma" + "@plasma//:plasma_client" ], ) @@ -165,7 +166,7 @@ cc_library( "src", ], deps = [ - "@plasma", + "@plasma//:plasma_client", ":sha256" ] ) @@ -304,3 +305,14 @@ flatbuffer_cc_library( flatc_args = FLATC_ARGS, out_prefix = "src/ray/object_manager/format/", ) + +pyx_library( + name = "_raylet", + srcs = glob([ + "python/ray/__init__.py", + "python/ray/_raylet.pyx", + "python/ray/includes/*.pxd", + "python/ray/includes/*.pxi" + ]), + deps = ["//:raylet_lib"] +) diff --git a/WORKSPACE b/WORKSPACE index c44b45e89..2074dcdf1 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -25,5 +25,15 @@ new_git_repository( name = "plasma", build_file = "@//bazel:BUILD.plasma", remote = "https://github.com/ray-project/arrow", - commit = "1e4f867eb1dc31107331ab1defdffb94467f31dc", + commit = "f5d1be2fed69899aea636bd074aaeaa4149acc79", ) + +new_git_repository( + name="cython", + build_file="@//bazel:BUILD.cython", + remote = "https://github.com/cython/cython", + commit = "49414dbc7ddc2ca2979d6dbe1e44714b10d72e7e", +) + +load("@//bazel:python_configure.bzl", "python_configure") +python_configure(name="local_config_python") diff --git a/bazel/BUILD.cython b/bazel/BUILD.cython new file mode 100644 index 000000000..c40a733d0 --- /dev/null +++ b/bazel/BUILD.cython @@ -0,0 +1,30 @@ +# Adapted from grpc/third_party/cython.BUILD + +# Adapted with modifications from tensorflow/third_party/cython.BUILD + +py_library( + name="cython_lib", + srcs=glob( + ["Cython/**/*.py"], + exclude=[ + "**/Tests/*.py", + ], + ) + ["cython.py"], + data=glob([ + "Cython/**/*.pyx", + "Cython/Utility/*.*", + "Cython/Includes/**/*.pxd", + ]), + srcs_version="PY2AND3", + visibility=["//visibility:public"], +) + +# May not be named "cython", since that conflicts with Cython/ on OSX +py_binary( + name="cython_binary", + srcs=["cython.py"], + main="cython.py", + srcs_version="PY2AND3", + visibility=["//visibility:public"], + deps=["cython_lib"], +) diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index 23bbf3b11..be6c6d55f 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -1,28 +1,70 @@ load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library") cc_library( - name = "plasma", + name = "arrow", + hdrs = [ + "cpp/src/arrow/buffer.h", + "cpp/src/arrow/memory_pool.h", + "cpp/src/arrow/status.h", + "cpp/src/arrow/io/interfaces.h", + "cpp/src/arrow/util/bit-util.h", + "cpp/src/arrow/util/io-util.h", + "cpp/src/arrow/util/logging.h", + "cpp/src/arrow/util/macros.h", + "cpp/src/arrow/util/string_builder.h", + "cpp/src/arrow/util/string_view.h", + "cpp/src/arrow/util/type_traits.h", + "cpp/src/arrow/util/thread-pool.h", + "cpp/src/arrow/util/visibility.h", + "cpp/src/arrow/util/windows_compatibility.h", + "cpp/src/arrow/vendored/string_view.hpp", + "cpp/src/arrow/vendored/xxhash/xxhash.h", + "cpp/src/arrow/vendored/xxhash/xxhash.c", + ], + srcs = [ + "cpp/src/arrow/buffer.cc", + "cpp/src/arrow/memory_pool.cc", + "cpp/src/arrow/status.cc", + "cpp/src/arrow/util/logging.cc", + "cpp/src/arrow/util/thread-pool.cc", + "cpp/src/arrow/util/io-util.cc", + ], + strip_include_prefix = "cpp/src", +) + +cc_library( + name = "plasma_client", visibility = ["//visibility:public"], hdrs = [ "cpp/src/plasma/client.h", "cpp/src/plasma/common.h", "cpp/src/plasma/compat.h", - "cpp/src/plasma/events.h", - "cpp/src/arrow/buffer.h", - "cpp/src/arrow/memory_pool.h", - "cpp/src/arrow/status.h", - "cpp/src/arrow/util/macros.h", - "cpp/src/arrow/util/visibility.h", - "cpp/src/arrow/util/string_builder.h", - # private headers - "cpp/src/plasma/common_generated.h", - "cpp/src/plasma/plasma_generated.h", - "cpp/src/plasma/eviction_policy.h", "cpp/src/plasma/fling.h", "cpp/src/plasma/io.h", "cpp/src/plasma/malloc.h", "cpp/src/plasma/plasma.h", "cpp/src/plasma/protocol.h", + "cpp/src/plasma/common_generated.h", + "cpp/src/plasma/plasma_generated.h", + ], + srcs = [ + "cpp/src/plasma/client.cc", + "cpp/src/plasma/common.cc", + "cpp/src/plasma/fling.cc", + "cpp/src/plasma/io.cc", + "cpp/src/plasma/malloc.cc", + "cpp/src/plasma/plasma.cc", + "cpp/src/plasma/protocol.cc", + ], + deps = [":common_fbs", ":plasma_fbs", ":arrow"], + strip_include_prefix = "cpp/src", +) + +cc_library( + name = "plasma_lib", + hdrs = [ + "cpp/src/plasma/events.h", + "cpp/src/plasma/eviction_policy.h", "cpp/src/plasma/store.h", "cpp/src/plasma/thirdparty/dlmalloc.c", "cpp/src/plasma/thirdparty/ae/ae.h", @@ -32,32 +74,27 @@ cc_library( "cpp/src/plasma/thirdparty/ae/ae_select.c", "cpp/src/plasma/thirdparty/ae/config.h", "cpp/src/plasma/thirdparty/ae/zmalloc.h", - "cpp/src/arrow/io/interfaces.h", - "cpp/src/arrow/util/bit-util.h", - "cpp/src/arrow/util/io-util.h", - "cpp/src/arrow/util/logging.h", - "cpp/src/arrow/util/string_view.h", - "cpp/src/arrow/util/type_traits.h", - "cpp/src/arrow/vendored/string_view.hpp", - "cpp/src/arrow/util/thread-pool.h", - "cpp/src/arrow/vendored/xxhash/xxhash.h", - "cpp/src/arrow/vendored/xxhash/xxhash.c", - "cpp/src/arrow/util/windows_compatibility.h", ], - srcs = glob([ - "cpp/src/plasma/*.cc", - "cpp/src/plasma/thirdparty/ae/ae.c", - "cpp/src/arrow/buffer.cc", - "cpp/src/arrow/memory_pool.cc", - "cpp/src/arrow/status.cc", - "cpp/src/arrow/util/logging.cc", - "cpp/src/arrow/util/thread-pool.cc", - "cpp/src/arrow/util/io-util.cc", - ]), - deps = [":common_fbs", ":plasma_fbs"], + srcs = [ + "cpp/src/plasma/dlmalloc.cc", + "cpp/src/plasma/eviction_policy.cc", + "cpp/src/plasma/events.cc", + ], + deps = [":plasma_client"], strip_include_prefix = "cpp/src", ) + +cc_binary( + name = "plasma_server", + visibility = ["//visibility:public"], + srcs = glob([ + "cpp/src/plasma/store.cc", + "cpp/src/plasma/thirdparty/ae/ae.c", + ], exclude = ["cpp/src/plasma/dlmalloc.cc", "cpp/src/plasma/events.cc"]), + deps = [":plasma_lib"], +) + FLATC_ARGS = [ "--gen-object-api", "--gen-mutable", diff --git a/bazel/BUILD.tpl b/bazel/BUILD.tpl new file mode 100644 index 000000000..257045c2e --- /dev/null +++ b/bazel/BUILD.tpl @@ -0,0 +1,36 @@ +# Adapted from grpc/third_party/py/BUILD.tpl + +# Adapted with modifications from tensorflow/third_party/py/ + +package(default_visibility=["//visibility:public"]) + +# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib +# See https://docs.python.org/3/extending/windows.html +cc_import( + name="python_lib", + interface_library=select({ + ":windows": ":python_import_lib", + # A placeholder for Unix platforms which makes --no_build happy. + "//conditions:default": "not-existing.lib", + }), + system_provided=1, +) + +cc_library( + name="python_headers", + hdrs=[":python_include"], + deps=select({ + ":windows": [":python_lib"], + "//conditions:default": [], + }), + includes=["python_include"], +) + +config_setting( + name="windows", + values={"cpu": "x64_windows"}, + visibility=["//visibility:public"], +) + +%{PYTHON_INCLUDE_GENRULE} +%{PYTHON_IMPORT_LIB_GENRULE} diff --git a/bazel/cython_library.bzl b/bazel/cython_library.bzl new file mode 100644 index 000000000..808744f74 --- /dev/null +++ b/bazel/cython_library.bzl @@ -0,0 +1,72 @@ +"""Custom rules for gRPC Python""" + +# Adapted from grpc/bazel/cython_library.bzl + +# Adapted with modifications from +# tensorflow/tensorflow/core/platform/default/build_config.bzl +# Native Bazel rules don't exist yet to compile Cython code, but rules have +# been written at cython/cython and tensorflow/tensorflow. We branch from +# Tensorflow's version as it is more actively maintained and works for gRPC +# Python's needs. +def pyx_library(name, deps=[], py_deps=[], srcs=[], **kwargs): + """Compiles a group of .pyx / .pxd / .py files. + First runs Cython to create .cpp files for each input .pyx or .py + .pxd + pair. Then builds a shared object for each, passing "deps" to each cc_binary + rule (includes Python headers by default). Finally, creates a py_library rule + with the shared objects and any pure Python "srcs", with py_deps as its + dependencies; the shared objects can be imported like normal Python files. + Args: + name: Name for the rule. + deps: C/C++ dependencies of the Cython (e.g. Numpy headers). + py_deps: Pure Python dependencies of the final library. + srcs: .py, .pyx, or .pxd files to either compile or pass through. + **kwargs: Extra keyword arguments passed to the py_library. + """ + # First filter out files that should be run compiled vs. passed through. + py_srcs = [] + pyx_srcs = [] + pxd_srcs = [] + for src in srcs: + if src.endswith(".pyx") or (src.endswith(".py") and + src[:-3] + ".pxd" in srcs): + pyx_srcs.append(src) + elif src.endswith(".py"): + py_srcs.append(src) + else: + pxd_srcs.append(src) + if src.endswith("__init__.py"): + pxd_srcs.append(src) + + # Invoke cython to produce the shared object libraries. + for filename in pyx_srcs: + native.genrule( + name=filename + "_cython_translation", + srcs=[filename], + outs=[filename.split(".")[0] + ".cpp"], + # Optionally use PYTHON_BIN_PATH on Linux platforms so that python 3 + # works. Windows has issues with cython_binary so skip PYTHON_BIN_PATH. + cmd= + "PYTHONHASHSEED=0 $(location @cython//:cython_binary) --cplus $(SRCS) --output-file $(OUTS)", + tools=["@cython//:cython_binary"] + pxd_srcs, + ) + + shared_objects = [] + for src in pyx_srcs: + stem = src.split(".")[0] + shared_object_name = stem + ".so" + native.cc_binary( + name=shared_object_name, + srcs=[stem + ".cpp"], + deps=deps + ["@local_config_python//:python_headers"], + linkshared=1, + ) + shared_objects.append(shared_object_name) + + # Now create a py_library with these shared objects as data. + native.py_library( + name=name, + srcs=py_srcs, + deps=py_deps, + srcs_version="PY2AND3", + data=shared_objects, + **kwargs) diff --git a/bazel/python_configure.bzl b/bazel/python_configure.bzl new file mode 100644 index 000000000..547f30bcf --- /dev/null +++ b/bazel/python_configure.bzl @@ -0,0 +1,299 @@ +# Adapted with a minor modification from grpc/third_party/py/python_configure.bzl: +# Changed "//third_party/py:%s.tpl" -> "@//bazel:%s.tpl" + +# Adapted with modifications from tensorflow/third_party/py/ + +"""Repository rule for Python autoconfiguration. +`python_configure` depends on the following environment variables: + * `PYTHON_BIN_PATH`: location of python binary. + * `PYTHON_LIB_PATH`: Location of python libraries. +""" + +_BAZEL_SH = "BAZEL_SH" +_PYTHON_BIN_PATH = "PYTHON_BIN_PATH" +_PYTHON_LIB_PATH = "PYTHON_LIB_PATH" +_PYTHON_CONFIG_REPO = "PYTHON_CONFIG_REPO" + + +def _tpl(repository_ctx, tpl, substitutions={}, out=None): + if not out: + out = tpl + repository_ctx.template(out, Label("@//bazel:%s.tpl" % tpl), + substitutions) + + +def _fail(msg): + """Output failure message when auto configuration fails.""" + red = "\033[0;31m" + no_color = "\033[0m" + fail("%sPython Configuration Error:%s %s\n" % (red, no_color, msg)) + + +def _is_windows(repository_ctx): + """Returns true if the host operating system is windows.""" + os_name = repository_ctx.os.name.lower() + return os_name.find("windows") != -1 + + +def _execute(repository_ctx, + cmdline, + error_msg=None, + error_details=None, + empty_stdout_fine=False): + """Executes an arbitrary shell command. + Args: + repository_ctx: the repository_ctx object + cmdline: list of strings, the command to execute + error_msg: string, a summary of the error if the command fails + error_details: string, details about the error or steps to fix it + empty_stdout_fine: bool, if True, an empty stdout result is fine, otherwise + it's an error + Return: + the result of repository_ctx.execute(cmdline) + """ + result = repository_ctx.execute(cmdline) + if result.stderr or not (empty_stdout_fine or result.stdout): + _fail("\n".join([ + error_msg.strip() if error_msg else "Repository command failed", + result.stderr.strip(), error_details if error_details else "" + ])) + else: + return result + + +def _read_dir(repository_ctx, src_dir): + """Returns a string with all files in a directory. + Finds all files inside a directory, traversing subfolders and following + symlinks. The returned string contains the full path of all files + separated by line breaks. + """ + if _is_windows(repository_ctx): + src_dir = src_dir.replace("/", "\\") + find_result = _execute( + repository_ctx, + ["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"], + empty_stdout_fine=True) + # src_files will be used in genrule.outs where the paths must + # use forward slashes. + return find_result.stdout.replace("\\", "/") + else: + find_result = _execute( + repository_ctx, ["find", src_dir, "-follow", "-type", "f"], + empty_stdout_fine=True) + return find_result.stdout + + +def _genrule(src_dir, genrule_name, command, outs): + """Returns a string with a genrule. + Genrule executes the given command and produces the given outputs. + """ + return ('genrule(\n' + ' name = "' + genrule_name + '",\n' + + ' outs = [\n' + outs + '\n ],\n' + ' cmd = """\n' + + command + '\n """,\n' + ')\n') + + +def _normalize_path(path): + """Returns a path with '/' and remove the trailing slash.""" + path = path.replace("\\", "/") + if path[-1] == "/": + path = path[:-1] + return path + + +def _symlink_genrule_for_dir(repository_ctx, + src_dir, + dest_dir, + genrule_name, + src_files=[], + dest_files=[]): + """Returns a genrule to symlink(or copy if on Windows) a set of files. + If src_dir is passed, files will be read from the given directory; otherwise + we assume files are in src_files and dest_files + """ + if src_dir != None: + src_dir = _normalize_path(src_dir) + dest_dir = _normalize_path(dest_dir) + files = '\n'.join( + sorted(_read_dir(repository_ctx, src_dir).splitlines())) + # Create a list with the src_dir stripped to use for outputs. + dest_files = files.replace(src_dir, '').splitlines() + src_files = files.splitlines() + command = [] + outs = [] + for i in range(len(dest_files)): + if dest_files[i] != "": + # If we have only one file to link we do not want to use the dest_dir, as + # $(@D) will include the full path to the file. + dest = '$(@D)/' + dest_dir + dest_files[i] if len( + dest_files) != 1 else '$(@D)/' + dest_files[i] + # On Windows, symlink is not supported, so we just copy all the files. + cmd = 'cp -f' if _is_windows(repository_ctx) else 'ln -s' + command.append(cmd + ' "%s" "%s"' % (src_files[i], dest)) + outs.append(' "' + dest_dir + dest_files[i] + '",') + return _genrule(src_dir, genrule_name, " && ".join(command), + "\n".join(outs)) + + +def _get_python_bin(repository_ctx): + """Gets the python bin path.""" + python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH) + if python_bin != None: + return python_bin + python_bin_path = repository_ctx.which("python") + if python_bin_path != None: + return str(python_bin_path) + _fail("Cannot find python in PATH, please make sure " + + "python is installed and add its directory in PATH, or --define " + + "%s='/something/else'.\nPATH=%s" % + (_PYTHON_BIN_PATH, repository_ctx.os.environ.get("PATH", ""))) + + +def _get_bash_bin(repository_ctx): + """Gets the bash bin path.""" + bash_bin = repository_ctx.os.environ.get(_BAZEL_SH) + if bash_bin != None: + return bash_bin + else: + bash_bin_path = repository_ctx.which("bash") + if bash_bin_path != None: + return str(bash_bin_path) + else: + _fail( + "Cannot find bash in PATH, please make sure " + + "bash is installed and add its directory in PATH, or --define " + + "%s='/path/to/bash'.\nPATH=%s" % + (_BAZEL_SH, repository_ctx.os.environ.get("PATH", ""))) + + +def _get_python_lib(repository_ctx, python_bin): + """Gets the python lib path.""" + python_lib = repository_ctx.os.environ.get(_PYTHON_LIB_PATH) + if python_lib != None: + return python_lib + print_lib = ( + "<=1:\n" + + " print(paths[0])\n" + "END") + cmd = '%s - %s' % (python_bin, print_lib) + result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) + return result.stdout.strip('\n') + + +def _check_python_lib(repository_ctx, python_lib): + """Checks the python lib path.""" + cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib) + result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) + if result.return_code == 1: + _fail("Invalid python library path: %s" % python_lib) + + +def _check_python_bin(repository_ctx, python_bin): + """Checks the python bin path.""" + cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin) + result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) + if result.return_code == 1: + _fail("--define %s='%s' is not executable. Is it the python binary?" % + (_PYTHON_BIN_PATH, python_bin)) + + +def _get_python_include(repository_ctx, python_bin): + """Gets the python include path.""" + result = _execute( + repository_ctx, [ + python_bin, "-c", 'from __future__ import print_function;' + + 'from distutils import sysconfig;' + + 'print(sysconfig.get_python_inc())' + ], + error_msg="Problem getting python include path.", + error_details=( + "Is the Python binary path set up right? " + "(See ./configure or " + + _PYTHON_BIN_PATH + ".) " + "Is distutils installed?")) + return result.stdout.splitlines()[0] + + +def _get_python_import_lib_name(repository_ctx, python_bin): + """Get Python import library name (pythonXY.lib) on Windows.""" + result = _execute( + repository_ctx, [ + python_bin, "-c", + 'import sys;' + 'print("python" + str(sys.version_info[0]) + ' + + ' str(sys.version_info[1]) + ".lib")' + ], + error_msg="Problem getting python import library.", + error_details=("Is the Python binary path set up right? " + + "(See ./configure or " + _PYTHON_BIN_PATH + ".) ")) + return result.stdout.splitlines()[0] + + +def _create_local_python_repository(repository_ctx): + """Creates the repository containing files set up to build with Python.""" + python_bin = _get_python_bin(repository_ctx) + _check_python_bin(repository_ctx, python_bin) + python_lib = _get_python_lib(repository_ctx, python_bin) + _check_python_lib(repository_ctx, python_lib) + python_include = _get_python_include(repository_ctx, python_bin) + python_include_rule = _symlink_genrule_for_dir( + repository_ctx, python_include, 'python_include', 'python_include') + python_import_lib_genrule = "" + # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib + # See https://docs.python.org/3/extending/windows.html + if _is_windows(repository_ctx): + python_include = _normalize_path(python_include) + python_import_lib_name = _get_python_import_lib_name( + repository_ctx, python_bin) + python_import_lib_src = python_include.rsplit( + '/', 1)[0] + "/libs/" + python_import_lib_name + python_import_lib_genrule = _symlink_genrule_for_dir( + repository_ctx, None, '', 'python_import_lib', + [python_import_lib_src], [python_import_lib_name]) + _tpl( + repository_ctx, "BUILD", { + "%{PYTHON_INCLUDE_GENRULE}": python_include_rule, + "%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule, + }) + + +def _create_remote_python_repository(repository_ctx, remote_config_repo): + """Creates pointers to a remotely configured repo set up to build with Python. + """ + _tpl(repository_ctx, "remote.BUILD", { + "%{REMOTE_PYTHON_REPO}": remote_config_repo, + }, "BUILD") + + +def _python_autoconf_impl(repository_ctx): + """Implementation of the python_autoconf repository rule.""" + if _PYTHON_CONFIG_REPO in repository_ctx.os.environ: + _create_remote_python_repository( + repository_ctx, repository_ctx.os.environ[_PYTHON_CONFIG_REPO]) + else: + _create_local_python_repository(repository_ctx) + + +python_configure = repository_rule( + implementation=_python_autoconf_impl, + environ=[ + _BAZEL_SH, + _PYTHON_BIN_PATH, + _PYTHON_LIB_PATH, + _PYTHON_CONFIG_REPO, + ], +) +"""Detects and configures the local Python. +Add the following to your WORKSPACE FILE: +```python +python_configure(name = "local_config_python") +``` +Args: + name: A unique name for this workspace rule. +""" diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index d0cbd825a..818cb6820 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -11,6 +11,13 @@ from libcpp.utility cimport pair from libcpp.unordered_map cimport unordered_map from libcpp.vector cimport vector as c_vector +from cython.operator import dereference, postincrement +cimport cpython + +include "includes/unique_ids.pxi" +include "includes/ray_config.pxi" +include "includes/task.pxi" + from ray.includes.common cimport ( CUniqueID, CTaskID, CObjectID, CFunctionID, CActorClassID, CActorID, CActorHandleID, CWorkerID, CDriverID, CConfigID, CClientID, @@ -22,13 +29,6 @@ from ray.includes.task cimport CTaskSpecification from ray.includes.ray_config cimport RayConfig from ray.utils import decode -from cython.operator import dereference, postincrement -cimport cpython - -include "includes/unique_ids.pxi" -include "includes/ray_config.pxi" -include "includes/task.pxi" - if cpython.PY_MAJOR_VERSION >= 3: import pickle diff --git a/src/ray/gcs/client.cc b/src/ray/gcs/client.cc index ba660e4f0..c729856e4 100644 --- a/src/ray/gcs/client.cc +++ b/src/ray/gcs/client.cc @@ -158,12 +158,6 @@ AsyncGcsClient::AsyncGcsClient(const std::string &address, int port, AsyncGcsClient::AsyncGcsClient(const std::string &address, int port, bool is_test_client) : AsyncGcsClient(address, port, ClientID::from_random(), is_test_client) {} -Status Attach(plasma::EventLoop &event_loop) { - // TODO(pcm): Implement this via - // context()->AttachToEventLoop(event loop) - return Status::OK(); -} - Status AsyncGcsClient::Attach(boost::asio::io_service &io_service) { // Take care of sharding contexts. RAY_CHECK(shard_asio_async_clients_.empty()) << "Attach shall be called only once"; diff --git a/src/ray/gcs/client.h b/src/ray/gcs/client.h index e44355571..6145d0aa2 100644 --- a/src/ray/gcs/client.h +++ b/src/ray/gcs/client.h @@ -4,7 +4,6 @@ #include #include -#include "plasma/events.h" #include "ray/gcs/asio.h" #include "ray/gcs/tables.h" #include "ray/id.h" @@ -41,9 +40,6 @@ class RAY_EXPORT AsyncGcsClient { AsyncGcsClient(const std::string &address, int port, const std::string &password); AsyncGcsClient(const std::string &address, int port, bool is_test_client); - /// Attach this client to a plasma event loop. Note that only - /// one event loop should be attached at a time. - Status Attach(plasma::EventLoop &event_loop); /// Attach this client to an asio event loop. Note that only /// one event loop should be attached at a time. Status Attach(boost::asio::io_service &io_service); diff --git a/src/ray/object_manager/object_buffer_pool.h b/src/ray/object_manager/object_buffer_pool.h index d39f5b265..78814710a 100644 --- a/src/ray/object_manager/object_buffer_pool.h +++ b/src/ray/object_manager/object_buffer_pool.h @@ -11,7 +11,6 @@ #include #include "plasma/client.h" -#include "plasma/events.h" #include "ray/id.h" #include "ray/status.h" diff --git a/src/ray/object_manager/object_manager.h b/src/ray/object_manager/object_manager.h index 57cf27e8b..29c75c57a 100644 --- a/src/ray/object_manager/object_manager.h +++ b/src/ray/object_manager/object_manager.h @@ -14,7 +14,6 @@ #include #include "plasma/client.h" -#include "plasma/events.h" #include "ray/common/client_connection.h" #include "ray/id.h" diff --git a/src/ray/object_manager/object_store_notification_manager.h b/src/ray/object_manager/object_store_notification_manager.h index 45d131602..92db89573 100644 --- a/src/ray/object_manager/object_store_notification_manager.h +++ b/src/ray/object_manager/object_store_notification_manager.h @@ -10,7 +10,6 @@ #include #include "plasma/client.h" -#include "plasma/events.h" #include "ray/id.h" #include "ray/status.h"