wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-27 15:16:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

Use GRCP and Bazel 1.0 (#6002)

Browse Source
This commit is contained in:
Philipp Moritz
2019-11-08 15:58:28 -08:00
committed by GitHub
parent afca6d3d87
commit ccbcc4bafa
15 changed files with 66 additions and 471 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bazelrc
+5 -1
View File
@@ -1,7 +1,8 @@
# build config
build --compilation_mode=opt
build --action_env=PATH
build --action_env=PYTHON_BIN_PATH
build --action_env=PYTHON2_BIN_PATH
build --action_env=PYTHON3_BIN_PATH
# Warnings should be errors
build --per_file_copt=-src/ray/thirdparty/hiredis/dict.c,-.*/arrow/util/logging.cc@-Werror
# Ignore warnings for protobuf generated files and external projects.
@@ -10,3 +11,6 @@ build --per_file_copt='external*@-w'
# This workaround is needed due to https://github.com/bazelbuild/bazel/issues/4341
build --per_file_copt="external/com_github_grpc_grpc/.*@-DGRPC_BAZEL_BUILD"
build --http_timeout_scaling=5.0
# This workaround is due to an incompatibility of
# bazel_common/tools/maven/pom_file.bzl with Bazel 1.0
build --incompatible_depset_is_not_iterable=false
BUILD.bazel
+3 -4
View File
@@ -2,11 +2,11 @@
# C/C++ documentation: https://docs.bazel.build/versions/master/be/c-cpp.html
load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library")
load("@build_stack_rules_proto//python:python_proto_compile.bzl", "python_proto_compile")
load("@build_stack_rules_proto//python:python_grpc_compile.bzl", "python_grpc_compile")
load("@com_github_grpc_grpc//bazel:cython_library.bzl", "pyx_library")
load("@rules_proto_grpc//python:defs.bzl", "python_proto_compile")
load("@rules_proto_grpc//python:defs.bzl", "python_grpc_compile")
load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library")
load("@//bazel:ray.bzl", "flatbuffer_py_library")
load("@//bazel:cython_library.bzl", "pyx_library")
COPTS = ["-DRAY_USE_GLOG"] + select({
"@bazel_tools//src/conditions:windows": [
@@ -744,7 +744,6 @@ pyx_library(
"python/ray/includes/*.pxd",
"python/ray/includes/*.pxi",
]),
copts = COPTS,
deps = [
"//:core_worker_lib",
"//:raylet_lib",
WORKSPACE
+7
View File
@@ -7,3 +7,10 @@ ray_deps_setup()
load("//bazel:ray_deps_build_all.bzl", "ray_deps_build_all")
ray_deps_build_all()
# This needs to be run after grpc_deps() in ray_deps_build_all() to make
# sure all the packages loaded by grpc_deps() are available. However a
# load() statement cannot be in a function so we put it here.
load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps")
grpc_extra_deps()
bazel/BUILD.tpl
-36
View File
@@ -1,36 +0,0 @@
# Adapted from grpc/third_party/py/BUILD.tpl
# Adapted with modifications from tensorflow/third_party/py/
package(default_visibility=["//visibility:public"])
# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
# See https://docs.python.org/3/extending/windows.html
cc_import(
name="python_lib",
interface_library=select({
":windows": ":python_import_lib",
# A placeholder for Unix platforms which makes --no_build happy.
"//conditions:default": "not-existing.lib",
}),
system_provided=1,
)
cc_library(
name="python_headers",
hdrs=[":python_include"],
deps=select({
":windows": [":python_lib"],
"//conditions:default": [],
}),
includes=["python_include"],
)
config_setting(
name="windows",
values={"cpu": "x64_windows"},
visibility=["//visibility:public"],
)
%{PYTHON_INCLUDE_GENRULE}
%{PYTHON_IMPORT_LIB_GENRULE}
bazel/cython_library.bzl
-73
View File
@@ -1,73 +0,0 @@
"""Custom rules for gRPC Python"""
# Adapted from grpc/bazel/cython_library.bzl
# Adapted with modifications from
# tensorflow/tensorflow/core/platform/default/build_config.bzl
# Native Bazel rules don't exist yet to compile Cython code, but rules have
# been written at cython/cython and tensorflow/tensorflow. We branch from
# Tensorflow's version as it is more actively maintained and works for gRPC
# Python's needs.
def pyx_library(name, deps=[], py_deps=[], srcs=[], copts=[], **kwargs):
"""Compiles a group of .pyx / .pxd / .py files.
First runs Cython to create .cpp files for each input .pyx or .py + .pxd
pair. Then builds a shared object for each, passing "deps" to each cc_binary
rule (includes Python headers by default). Finally, creates a py_library rule
with the shared objects and any pure Python "srcs", with py_deps as its
dependencies; the shared objects can be imported like normal Python files.
Args:
name: Name for the rule.
deps: C/C++ dependencies of the Cython (e.g. Numpy headers).
py_deps: Pure Python dependencies of the final library.
srcs: .py, .pyx, or .pxd files to either compile or pass through.
**kwargs: Extra keyword arguments passed to the py_library.
"""
# First filter out files that should be run compiled vs. passed through.
py_srcs = []
pyx_srcs = []
pxd_srcs = []
for src in srcs:
if src.endswith(".pyx") or (src.endswith(".py") and
src[:-3] + ".pxd" in srcs):
pyx_srcs.append(src)
elif src.endswith(".py"):
py_srcs.append(src)
else:
pxd_srcs.append(src)
if src.endswith("__init__.py"):
pxd_srcs.append(src)
# Invoke cython to produce the shared object libraries.
for filename in pyx_srcs:
native.genrule(
name=filename + "_cython_translation",
srcs=[filename],
outs=[filename.split(".")[0] + ".cpp"],
# Optionally use PYTHON_BIN_PATH on Linux platforms so that python 3
# works. Windows has issues with cython_binary so skip PYTHON_BIN_PATH.
cmd=
'PYTHONHASHSEED=0 "$${PYTHON_BIN_PATH}" "$(location @cython//:cython_binary)" --cplus $(SRCS) --output-file $(OUTS)',
tools=["@cython//:cython_binary"] + pxd_srcs,
)
shared_objects = []
for src in pyx_srcs:
stem = src.split(".")[0]
shared_object_name = stem + ".so"
native.cc_binary(
name=shared_object_name,
srcs=[stem + ".cpp"],
copts=copts,
deps=deps + ["@local_config_python//:python_headers"],
linkshared=1,
)
shared_objects.append(shared_object_name)
# Now create a py_library with these shared objects as data.
native.py_library(
name=name,
srcs=py_srcs,
deps=py_deps,
srcs_version="PY2AND3",
data=shared_objects,
**kwargs)
bazel/python_configure.bzl
-299
View File
@@ -1,299 +0,0 @@
# Adapted with a minor modification from grpc/third_party/py/python_configure.bzl:
# Changed "//third_party/py:%s.tpl" -> "@//bazel:%s.tpl"
# Adapted with modifications from tensorflow/third_party/py/
"""Repository rule for Python autoconfiguration.
`python_configure` depends on the following environment variables:
* `PYTHON_BIN_PATH`: location of python binary.
* `PYTHON_LIB_PATH`: Location of python libraries.
"""
_BAZEL_SH = "BAZEL_SH"
_PYTHON_BIN_PATH = "PYTHON_BIN_PATH"
_PYTHON_LIB_PATH = "PYTHON_LIB_PATH"
_PYTHON_CONFIG_REPO = "PYTHON_CONFIG_REPO"
def _tpl(repository_ctx, tpl, substitutions={}, out=None):
if not out:
out = tpl
repository_ctx.template(out, Label("@//bazel:%s.tpl" % tpl),
substitutions)
def _fail(msg):
"""Output failure message when auto configuration fails."""
red = "\033[0;31m"
no_color = "\033[0m"
fail("%sPython Configuration Error:%s %s\n" % (red, no_color, msg))
def _is_windows(repository_ctx):
"""Returns true if the host operating system is windows."""
os_name = repository_ctx.os.name.lower()
return os_name.find("windows") != -1
def _execute(repository_ctx,
cmdline,
error_msg=None,
error_details=None,
empty_stdout_fine=False):
"""Executes an arbitrary shell command.
Args:
repository_ctx: the repository_ctx object
cmdline: list of strings, the command to execute
error_msg: string, a summary of the error if the command fails
error_details: string, details about the error or steps to fix it
empty_stdout_fine: bool, if True, an empty stdout result is fine, otherwise
it's an error
Return:
the result of repository_ctx.execute(cmdline)
"""
result = repository_ctx.execute(cmdline)
if result.stderr or not (empty_stdout_fine or result.stdout):
_fail("\n".join([
error_msg.strip() if error_msg else "Repository command failed",
result.stderr.strip(), error_details if error_details else ""
]))
else:
return result
def _read_dir(repository_ctx, src_dir):
"""Returns a string with all files in a directory.
Finds all files inside a directory, traversing subfolders and following
symlinks. The returned string contains the full path of all files
separated by line breaks.
"""
if _is_windows(repository_ctx):
src_dir = src_dir.replace("/", "\\")
find_result = _execute(
repository_ctx,
["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"],
empty_stdout_fine=True)
# src_files will be used in genrule.outs where the paths must
# use forward slashes.
return find_result.stdout.replace("\\", "/")
else:
find_result = _execute(
repository_ctx, ["find", src_dir, "-follow", "-type", "f"],
empty_stdout_fine=True)
return find_result.stdout
def _genrule(src_dir, genrule_name, command, outs):
"""Returns a string with a genrule.
Genrule executes the given command and produces the given outputs.
"""
return ('genrule(\n' + ' name = "' + genrule_name + '",\n' +
' outs = [\n' + outs + '\n ],\n' + ' cmd = """\n' +
command + '\n """,\n' + ')\n')
def _normalize_path(path):
"""Returns a path with '/' and remove the trailing slash."""
path = path.replace("\\", "/")
if path[-1] == "/":
path = path[:-1]
return path
def _symlink_genrule_for_dir(repository_ctx,
src_dir,
dest_dir,
genrule_name,
src_files=[],
dest_files=[]):
"""Returns a genrule to symlink(or copy if on Windows) a set of files.
If src_dir is passed, files will be read from the given directory; otherwise
we assume files are in src_files and dest_files
"""
if src_dir != None:
src_dir = _normalize_path(src_dir)
dest_dir = _normalize_path(dest_dir)
files = '\n'.join(
sorted(_read_dir(repository_ctx, src_dir).splitlines()))
# Create a list with the src_dir stripped to use for outputs.
dest_files = files.replace(src_dir, '').splitlines()
src_files = files.splitlines()
command = []
outs = []
for i in range(len(dest_files)):
if dest_files[i] != "":
# If we have only one file to link we do not want to use the dest_dir, as
# $(@D) will include the full path to the file.
dest = '$(@D)/' + dest_dir + dest_files[i] if len(
dest_files) != 1 else '$(@D)/' + dest_files[i]
# On Windows, symlink is not supported, so we just copy all the files.
cmd = 'cp -f' if _is_windows(repository_ctx) else 'ln -s'
command.append(cmd + ' "%s" "%s"' % (src_files[i], dest))
outs.append(' "' + dest_dir + dest_files[i] + '",')
return _genrule(src_dir, genrule_name, " && ".join(command),
"\n".join(outs))
def _get_python_bin(repository_ctx):
"""Gets the python bin path."""
python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH)
if python_bin != None:
return python_bin
python_bin_path = repository_ctx.which("python")
if python_bin_path != None:
return str(python_bin_path)
_fail("Cannot find python in PATH, please make sure " +
"python is installed and add its directory in PATH, or --define " +
"%s='/something/else'.\nPATH=%s" %
(_PYTHON_BIN_PATH, repository_ctx.os.environ.get("PATH", "")))
def _get_bash_bin(repository_ctx):
"""Gets the bash bin path."""
bash_bin = repository_ctx.os.environ.get(_BAZEL_SH)
if bash_bin != None:
return bash_bin
else:
bash_bin_path = repository_ctx.which("bash")
if bash_bin_path != None:
return str(bash_bin_path)
else:
_fail(
"Cannot find bash in PATH, please make sure " +
"bash is installed and add its directory in PATH, or --define "
+ "%s='/path/to/bash'.\nPATH=%s" %
(_BAZEL_SH, repository_ctx.os.environ.get("PATH", "")))
def _get_python_lib(repository_ctx, python_bin):
"""Gets the python lib path."""
python_lib = repository_ctx.os.environ.get(_PYTHON_LIB_PATH)
if python_lib != None:
return python_lib
print_lib = (
"<<END\n" + "from __future__ import print_function\n" +
"import site\n" + "import os\n" + "\n" + "try:\n" +
" input = raw_input\n" + "except NameError:\n" + " pass\n" + "\n" +
"python_paths = []\n" + "if os.getenv('PYTHONPATH') is not None:\n" +
" python_paths = os.getenv('PYTHONPATH').split(':')\n" + "try:\n" +
" library_paths = site.getsitepackages()\n" +
"except AttributeError:\n" +
" from distutils.sysconfig import get_python_lib\n" +
" library_paths = [get_python_lib()]\n" +
"all_paths = set(python_paths + library_paths)\n" + "paths = []\n" +
"for path in all_paths:\n" + " if os.path.isdir(path):\n" +
" paths.append(path)\n" + "if len(paths) >=1:\n" +
" print(paths[0])\n" + "END")
cmd = '"%s" - %s' % (python_bin, print_lib)
result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd])
return result.stdout.strip('\n')
def _check_python_lib(repository_ctx, python_lib):
"""Checks the python lib path."""
cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib)
result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd])
if result.return_code == 1:
_fail("Invalid python library path: %s" % python_lib)
def _check_python_bin(repository_ctx, python_bin):
"""Checks the python bin path."""
cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin)
result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd])
if result.return_code == 1:
_fail("--define %s='%s' is not executable. Is it the python binary?" %
(_PYTHON_BIN_PATH, python_bin))
def _get_python_include(repository_ctx, python_bin):
"""Gets the python include path."""
result = _execute(
repository_ctx, [
python_bin, "-c", 'from __future__ import print_function;' +
'from distutils import sysconfig;' +
'print(sysconfig.get_python_inc())'
],
error_msg="Problem getting python include path.",
error_details=(
"Is the Python binary path set up right? " + "(See ./configure or "
+ _PYTHON_BIN_PATH + ".) " + "Is distutils installed?"))
return result.stdout.splitlines()[0]
def _get_python_import_lib_name(repository_ctx, python_bin):
"""Get Python import library name (pythonXY.lib) on Windows."""
result = _execute(
repository_ctx, [
python_bin, "-c",
'import sys;' + 'print("python" + str(sys.version_info[0]) + ' +
' str(sys.version_info[1]) + ".lib")'
],
error_msg="Problem getting python import library.",
error_details=("Is the Python binary path set up right? " +
"(See ./configure or " + _PYTHON_BIN_PATH + ".) "))
return result.stdout.splitlines()[0]
def _create_local_python_repository(repository_ctx):
"""Creates the repository containing files set up to build with Python."""
python_bin = _get_python_bin(repository_ctx)
_check_python_bin(repository_ctx, python_bin)
python_lib = _get_python_lib(repository_ctx, python_bin)
_check_python_lib(repository_ctx, python_lib)
python_include = _get_python_include(repository_ctx, python_bin)
python_include_rule = _symlink_genrule_for_dir(
repository_ctx, python_include, 'python_include', 'python_include')
python_import_lib_genrule = ""
# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
# See https://docs.python.org/3/extending/windows.html
if _is_windows(repository_ctx):
python_include = _normalize_path(python_include)
python_import_lib_name = _get_python_import_lib_name(
repository_ctx, python_bin)
python_import_lib_src = python_include.rsplit(
'/', 1)[0] + "/libs/" + python_import_lib_name
python_import_lib_genrule = _symlink_genrule_for_dir(
repository_ctx, None, '', 'python_import_lib',
[python_import_lib_src], [python_import_lib_name])
_tpl(
repository_ctx, "BUILD", {
"%{PYTHON_INCLUDE_GENRULE}": python_include_rule,
"%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule,
})
def _create_remote_python_repository(repository_ctx, remote_config_repo):
"""Creates pointers to a remotely configured repo set up to build with Python.
"""
_tpl(repository_ctx, "remote.BUILD", {
"%{REMOTE_PYTHON_REPO}": remote_config_repo,
}, "BUILD")
def _python_autoconf_impl(repository_ctx):
"""Implementation of the python_autoconf repository rule."""
if _PYTHON_CONFIG_REPO in repository_ctx.os.environ:
_create_remote_python_repository(
repository_ctx, repository_ctx.os.environ[_PYTHON_CONFIG_REPO])
else:
_create_local_python_repository(repository_ctx)
python_configure = repository_rule(
implementation=_python_autoconf_impl,
environ=[
_BAZEL_SH,
_PYTHON_BIN_PATH,
_PYTHON_LIB_PATH,
_PYTHON_CONFIG_REPO,
],
)
"""Detects and configures the local Python.
Add the following to your WORKSPACE FILE:
```python
python_configure(name = "local_config_python")
```
Args:
name: A unique name for this workspace rule.
"""
bazel/ray_deps_build_all.bzl
+3 -5
View File
@@ -1,11 +1,10 @@
load("@com_github_ray_project_ray//java:dependencies.bzl", "gen_java_deps")
load("@com_github_nelhage_rules_boost//:boost/boost.bzl", "boost_deps")
load("@com_github_jupp0r_prometheus_cpp//:repositories.bzl", "prometheus_cpp_repositories")
load("@com_github_ray_project_ray//bazel:python_configure.bzl", "python_configure")
load("@com_github_checkstyle_java//:repo.bzl", "checkstyle_deps")
load("@com_github_grpc_grpc//third_party/py:python_configure.bzl", "python_configure")
load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
load("@build_stack_rules_proto//java:deps.bzl", "java_proto_compile")
load("@build_stack_rules_proto//python:deps.bzl", "python_proto_compile")
load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_toolchains")
def ray_deps_build_all():
@@ -15,5 +14,4 @@ def ray_deps_build_all():
prometheus_cpp_repositories()
python_configure(name = "local_config_python")
grpc_deps()
java_proto_compile()
python_proto_compile()
rules_proto_grpc_toolchains()
bazel/ray_deps_setup.bzl
+30 -29
View File
@@ -2,9 +2,8 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_r
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
def ray_deps_setup():
RULES_JVM_EXTERNAL_TAG = "1.2"
RULES_JVM_EXTERNAL_SHA = "e5c68b87f750309a79f59c2b69ead5c3221ffa54ff9496306937bfa1c9c8c86b"
RULES_JVM_EXTERNAL_TAG = "2.10"
RULES_JVM_EXTERNAL_SHA = "1bbf2e48d07686707dd85357e9a94da775e1dbd7c464272b3664283c9c716d26"
http_archive(
name = "rules_jvm_external",
@@ -17,7 +16,7 @@ def ray_deps_setup():
name = "bazel_common",
strip_prefix = "bazel-common-f1115e0f777f08c3cdb115526c4e663005bec69b",
url = "https://github.com/google/bazel-common/archive/f1115e0f777f08c3cdb115526c4e663005bec69b.zip",
sha256 = "1e05a4791cc3470d3ecf7edb556f796b1d340359f1c4d293f175d4d0946cf84c",
sha256 = "1e05a4791cc3470d3ecf7edb556f796b1d340359f1c4d293f175d4d0946cf84c",
)
BAZEL_SKYLIB_TAG = "0.6.0"
@@ -26,49 +25,50 @@ def ray_deps_setup():
name = "bazel_skylib",
strip_prefix = "bazel-skylib-%s" % BAZEL_SKYLIB_TAG,
url = "https://github.com/bazelbuild/bazel-skylib/archive/%s.tar.gz" % BAZEL_SKYLIB_TAG,
sha256 = "eb5c57e4c12e68c0c20bc774bfbc60a568e800d025557bc4ea022c6479acc867",
sha256 = "eb5c57e4c12e68c0c20bc774bfbc60a568e800d025557bc4ea022c6479acc867",
)
git_repository(
name = "com_github_checkstyle_java",
commit = "85f37871ca03b9d3fee63c69c8107f167e24e77b",
remote = "https://github.com/ruifangChen/checkstyle_java",
shallow_since = "1552542575 +0800",
commit = "ef367030d1433877a3360bbfceca18a5d0791bdd",
remote = "https://github.com/ray-project/checkstyle_java",
shallow_since = "1573090990 -0800",
)
git_repository(
name = "com_github_nelhage_rules_boost",
commit = "5171b9724fbb39c5fdad37b9ca9b544e8858d8ac",
remote = "https://github.com/ray-project/rules_boost",
shallow_since = "1556014830 +0800",
shallow_since = "1556014830 +0800",
)
git_repository(
name = "com_github_google_flatbuffers",
commit = "63d51afd1196336a7d1f56a988091ef05deb1c62",
remote = "https://github.com/google/flatbuffers.git",
shallow_since = "1547755012 -0800",
shallow_since = "1547755012 -0800",
)
git_repository(
name = "com_google_googletest",
commit = "3306848f697568aacf4bcca330f6bdd5ce671899",
remote = "https://github.com/google/googletest",
shallow_since = "1534270723 -0700",
shallow_since = "1534270723 -0700",
)
git_repository(
name = "com_github_gflags_gflags",
remote = "https://github.com/gflags/gflags.git",
commit = "e171aa2d15ed9eb17054558e0b3a6a413bb01067",
shallow_since = "1541971260 +0000",
commit = "e171aa2d15ed9eb17054558e0b3a6a413bb01067",
shallow_since = "1541971260 +0000",
)
new_git_repository(
name = "com_github_google_glog",
build_file = "@//bazel:BUILD.glog",
commit = "5c576f78c49b28d89b23fbb1fc80f54c879ec02e",
commit = "96a2f23dca4cc7180821ca5f32e526314395d26a",
remote = "https://github.com/google/glog",
shallow_since = "1550458164 +0900",
)
new_git_repository(
@@ -76,6 +76,7 @@ def ray_deps_setup():
build_file = "@//bazel:BUILD.plasma",
commit = "86f34aa07e611787d9cc98c6a33b0a0a536dce57",
remote = "https://github.com/apache/arrow",
shallow_since = "1572492886 -0700",
)
new_git_repository(
@@ -83,14 +84,14 @@ def ray_deps_setup():
build_file = "@//bazel:BUILD.cython",
commit = "49414dbc7ddc2ca2979d6dbe1e44714b10d72e7e",
remote = "https://github.com/cython/cython",
shallow_since = "1547888711 +0100",
shallow_since = "1547888711 +0100",
)
http_archive(
name = "io_opencensus_cpp",
strip_prefix = "opencensus-cpp-3aa11f20dd610cb8d2f7c62e58d1e69196aadf11",
urls = ["https://github.com/census-instrumentation/opencensus-cpp/archive/3aa11f20dd610cb8d2f7c62e58d1e69196aadf11.zip"],
sha256 = "92eef77c44d01e8472f68a2f1329919a1bb59317a4bb1e4d76081ab5c13a56d6",
sha256 = "92eef77c44d01e8472f68a2f1329919a1bb59317a4bb1e4d76081ab5c13a56d6",
)
# OpenCensus depends on Abseil so we have to explicitly pull it in.
@@ -99,7 +100,7 @@ def ray_deps_setup():
name = "com_google_absl",
commit = "aa844899c937bde5d2b24f276b59997e5b668bde",
remote = "https://github.com/abseil/abseil-cpp.git",
shallow_since = "1565288385 -0400",
shallow_since = "1565288385 -0400",
)
# OpenCensus depends on jupp0r/prometheus-cpp
@@ -109,19 +110,19 @@ def ray_deps_setup():
# TODO(qwang): We should use the repository of `jupp0r` here when this PR
# `https://github.com/jupp0r/prometheus-cpp/pull/225` getting merged.
remote = "https://github.com/ray-project/prometheus-cpp.git",
)
http_archive(
name = "com_github_grpc_grpc",
urls = [
"https://github.com/grpc/grpc/archive/76a381869413834692b8ed305fbe923c0f9c4472.tar.gz",
],
strip_prefix = "grpc-76a381869413834692b8ed305fbe923c0f9c4472",
sha256 = "b5efbe086b9a00826a3f830094312e6d1647157b5a5e7954a8ac4179bce3de8b",
shallow_since = "1572744904 -0700"
)
git_repository(
name = "build_stack_rules_proto",
remote = "https://github.com/stackb/rules_proto.git",
commit = "d9a123032f8436dbc34069cfc3207f2810a494ee",
name = "com_github_grpc_grpc",
commit = "93e8830070e9afcbaa992c75817009ee3f4b63a0",
remote = "https://github.com/grpc/grpc.git",
shallow_since = "1571118670 -0700",
)
git_repository(
name = "rules_proto_grpc",
commit = "a74fef39c5fe636580083545f76d1eab74f6450d",
remote = "https://github.com/rules-proto-grpc/rules_proto_grpc.git",
shallow_since = "1571494564 +0100",
)
build.sh
+3 -1
View File
@@ -101,7 +101,9 @@ if [ -z "$SKIP_PYARROW_INSTALL" ]; then
--target="$ROOT_DIR/python/ray/pyarrow_files" pyarrow==0.14.0.RAY \
--find-links https://s3-us-west-2.amazonaws.com/arrow-wheels/3a11193d9530fe8ec7fdb98057f853b708f6f6ae/index.html
fi
export PYTHON_BIN_PATH="$PYTHON_EXECUTABLE"
export PYTHON3_BIN_PATH="$PYTHON_EXECUTABLE"
export PYTHON2_BIN_PATH="$PYTHON_EXECUTABLE"
if [ "$RAY_BUILD_JAVA" == "YES" ]; then
"$BAZEL_EXECUTABLE" build //java:all --verbose_failures
ci/suppress_output
+2 -2
View File
@@ -6,11 +6,11 @@ PID=$$
# Print output to avoid travis killing us
watchdog() {
for i in `seq 5 5 120`; do
for i in `seq 5 5 150`; do
sleep 300
echo "This command has been running for more than $i minutes..."
done
echo "Command timed out after 2h, dumping logs:"
echo "Command timed out after 2.5h, dumping logs:"
cat $TMPFILE
echo "TIMED OUT"
kill -SIGKILL $PID
ci/travis/install-bazel.sh
+1 -1
View File
@@ -16,7 +16,7 @@ else
exit 1
fi
URL="https://github.com/bazelbuild/bazel/releases/download/0.26.1/bazel-0.26.1-installer-${platform}-x86_64.sh"
URL="https://github.com/bazelbuild/bazel/releases/download/1.1.0/bazel-1.1.0-installer-${platform}-x86_64.sh"
wget -O install.sh $URL
chmod +x install.sh
./install.sh --user
java/BUILD.bazel
+1 -1
View File
@@ -1,5 +1,5 @@
load("//bazel:ray.bzl", "define_java_module")
load("@build_stack_rules_proto//java:java_proto_compile.bzl", "java_proto_compile")
load("@rules_proto_grpc//java:defs.bzl", "java_proto_compile")
exports_files([
"testng.xml",
src/ray/rpc/client_call.h
+4 -4
View File
@@ -62,7 +62,7 @@ class ClientCallImpl : public ClientCall {
ClientCallback<Reply> callback_;
/// The response reader.
std::unique_ptr<grpc::ClientAsyncResponseReader<Reply>> response_reader_;
std::unique_ptr<grpc_impl::ClientAsyncResponseReader<Reply>> response_reader_;
/// gRPC status of this request.
grpc::Status status_;
@@ -106,9 +106,9 @@ class ClientCallTag {
/// \tparam Request Type of the request message.
/// \tparam Reply Type of the reply message.
template <class GrpcService, class Request, class Reply>
using PrepareAsyncFunction = std::unique_ptr<grpc::ClientAsyncResponseReader<Reply>> (
GrpcService::Stub::*)(grpc::ClientContext *context, const Request &request,
grpc::CompletionQueue *cq);
using PrepareAsyncFunction =
std::unique_ptr<grpc_impl::ClientAsyncResponseReader<Reply>> (GrpcService::Stub::*)(
grpc::ClientContext *context, const Request &request, grpc::CompletionQueue *cq);
/// `ClientCallManager` is used to manage outgoing gRPC requests and the lifecycles of
/// `ClientCall` objects.
src/ray/rpc/server_call.h
+2 -2
View File
@@ -198,7 +198,7 @@ class ServerCallImpl : public ServerCall {
grpc::ServerContext context_;
/// The response writer.
grpc::ServerAsyncResponseWriter<Reply> response_writer_;
grpc_impl::ServerAsyncResponseWriter<Reply> response_writer_;
/// The event loop.
boost::asio::io_service &io_service_;
@@ -226,7 +226,7 @@ class ServerCallImpl : public ServerCall {
/// \tparam Reply Type of the reply message.
template <class GrpcService, class Request, class Reply>
using RequestCallFunction = void (GrpcService::AsyncService::*)(
grpc::ServerContext *, Request *, grpc::ServerAsyncResponseWriter<Reply> *,
grpc::ServerContext *, Request *, grpc_impl::ServerAsyncResponseWriter<Reply> *,
grpc::CompletionQueue *, grpc::ServerCompletionQueue *, void *);
/// Implementation of `ServerCallFactory`
src/ray/test/run_gcs_tests.sh
+5 -13
View File
@@ -9,18 +9,10 @@ set -x
bazel build "//:redis_gcs_client_test" "//:actor_state_accessor_test" "//:subscription_executor_test" "//:asio_test" "//:libray_redis_module.so"
# Start Redis.
if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then
./src/credis/redis/src/redis-server \
--loglevel warning \
--loadmodule ./src/credis/build/src/libmember.so \
--loadmodule ./src/ray/gcs/redis_module/libray_redis_module.so \
--port 6379 &
else
./bazel-genfiles/redis-server \
--loglevel warning \
--loadmodule ./bazel-bin/libray_redis_module.so \
--port 6379 &
fi
./bazel-bin/redis-server \
--loglevel warning \
--loadmodule ./bazel-bin/libray_redis_module.so \
--port 6379 &
sleep 1s
./bazel-bin/redis_gcs_client_test
@@ -28,5 +20,5 @@ sleep 1s
./bazel-bin/subscription_executor_test
./bazel-bin/asio_test
./bazel-genfiles/redis-cli -p 6379 shutdown
./bazel-bin/redis-cli -p 6379 shutdown
sleep 1s