wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-15 11:25:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into py39

Browse Source
This commit is contained in:
Akash Patel
2020-12-16 11:04:27 -05:00
committed by GitHub
parent bd866d926d 91878d18b5
commit 7d8a008aeb
489 changed files with 36844 additions and 8744 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.flake8
+23
View File
@@ -0,0 +1,23 @@
[flake8]
exclude =
python/ray/core/generated/
streaming/python/generated
doc/source/conf.py
python/ray/cloudpickle/
python/ray/thirdparty_files/
python/build/
python/.eggs/
max-line-length = 79
inline-quotes = "
ignore =
C408
E121
E123
E126
E226
E24
E704
W503
W504
W605
avoid-escape = no
.github/ISSUE_TEMPLATE/bug_report.md
+2 -2
View File
@@ -14,9 +14,9 @@ assignees: ''
*Ray version and other system information (Python version, TensorFlow version, OS):*
### Reproduction (REQUIRED)
Please provide a script that can be run to reproduce the issue. The script should have **no external library dependencies** (i.e., use fake or mock data / environments):
Please provide a short code snippet (less than 50 lines if possible) that can be copy-pasted to reproduce the issue. The snippet should have **no external library dependencies** (i.e., use fake or mock data / environments):
If we cannot run your script, we cannot fix your issue.
If the code snippet cannot be run by itself, the issue will be closed with "needs-repro-script".
- [ ] I have verified my script runs in a clean environment and reproduces the issue.
- [ ] I have verified the issue also occurs with the [latest wheels](https://docs.ray.io/en/master/installation.html).
.github/ISSUE_TEMPLATE/feature_request.md
+1 -1
View File
@@ -2,7 +2,7 @@
name: Feature request/Question
about: For feature requests or questions, post on https://discuss.ray.io/ instead!
title: ''
labels: enhancement, triage
labels: enhancement
assignees: ''
---
.gitignore
+2
View File
@@ -174,6 +174,8 @@ venv
.*.swp
*.swp
tags
tags.lock
tags.temp
# Emacs
.#*
.travis.yml
+5 -5
View File
@@ -401,7 +401,7 @@ matrix:
- PYTHON=3.6
- TF_VERSION=2.2.0
- TFP_VERSION=0.8
- TORCH_VERSION=1.6
- TORCH_VERSION=1.7
- PYTHONWARNINGS=ignore
install:
- . ./ci/travis/ci.sh init RAY_CI_TUNE_AFFECTED
@@ -421,7 +421,7 @@ matrix:
- PYTHON=3.6
- TF_VERSION=2.1.0
- TFP_VERSION=0.8
- TORCH_VERSION=1.5
- TORCH_VERSION=1.7
- PYTHONWARNINGS=ignore
install:
- . ./ci/travis/ci.sh init RAY_CI_SGD_AFFECTED
@@ -432,7 +432,7 @@ matrix:
# - ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,-pytorch,-py37 python/ray/util/sgd/...
- ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=tf,-pytorch,-py37 python/ray/util/sgd/...
- ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,pytorch,-py37 python/ray/util/sgd/...
- ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/util/xgboost/...
# - ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/util/xgboost/...
# Docs: Tests and examples.
- os: linux
@@ -441,7 +441,7 @@ matrix:
- PYTHON=3.6
- TF_VERSION=2.1.0
- TFP_VERSION=0.8
- TORCH_VERSION=1.5
- TORCH_VERSION=1.7
- PYTHONWARNINGS=ignore
install:
- . ./ci/travis/ci.sh init RAY_CI_PYTHON_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_DOC_AFFECTED
@@ -459,7 +459,7 @@ matrix:
- INSTALL_HOROVOD=1
- TF_VERSION=2.1.0
- TFP_VERSION=0.8
- TORCH_VERSION=1.5
- TORCH_VERSION=1.7
- PYTHONWARNINGS=ignore
install:
- . ./ci/travis/ci.sh init RAY_CI_TUNE_AFFECTED,RAY_CI_SGD_AFFECTED
BUILD.bazel
+20 -2
View File
@@ -33,6 +33,7 @@ cc_library(
]),
hdrs = glob([
"src/ray/rpc/*.h",
"src/ray/raylet_client/*.h",
]),
copts = COPTS,
strip_include_prefix = "src",
@@ -55,6 +56,9 @@ cc_grpc_library(
# Node manager server and client.
cc_library(
name = "node_manager_rpc",
srcs = glob([
"src/ray/rpc/node_manager/*.cc",
]),
hdrs = glob([
"src/ray/rpc/node_manager/*.h",
]),
@@ -809,6 +813,18 @@ cc_test(
],
)
cc_test(
name = "pull_manager_test",
srcs = [
"src/ray/object_manager/test/pull_manager_test.cc",
],
copts = COPTS,
deps = [
":raylet_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "push_manager_test",
srcs = [
@@ -864,11 +880,13 @@ cc_test(
)
cc_test(
name = "scheduling_resources_test",
srcs = ["src/ray/common/task/scheduling_resources_test.cc"],
name = "local_placement_group_manager_test",
srcs = ["src/ray/raylet/placement_group_resource_manager_test.cc"],
copts = COPTS,
deps = [
"gcs_test_util_lib",
"ray_common",
"raylet_lib",
"@com_google_googletest//:gtest_main",
],
)
README.rst
+6 -3
View File
@@ -6,6 +6,9 @@
.. image:: https://img.shields.io/badge/Ray-Join%20Slack-blue
:target: https://forms.gle/9TSdDYUgxYs8SA9e8
.. image:: https://img.shields.io/badge/Discuss-Ask%20Questions-blue
:target: https://discuss.ray.io/
|
@@ -155,7 +158,7 @@ RLlib Quick Start
.. code-block:: bash
pip install tensorflow # or tensorflow-gpu
pip install "ray[rllib]" # also recommended: ray[debug]
pip install "ray[rllib]"
.. code-block:: python
@@ -298,13 +301,13 @@ Getting Involved
----------------
- `Community Slack`_: Join our Slack workspace.
- `GitHub Discussions`_: For discussions about development, questions about usage, and feature requests.
- `Forum`_: For discussions about development, questions about usage, and feature requests.
- `GitHub Issues`_: For reporting bugs.
- `Twitter`_: Follow updates on Twitter.
- `Meetup Group`_: Join our meetup group.
- `StackOverflow`_: For questions about how to use Ray.
.. _`GitHub Discussions`: https://github.com/ray-project/ray/discussions
.. _`Forum`: https://discuss.ray.io/
.. _`GitHub Issues`: https://github.com/ray-project/ray/issues
.. _`StackOverflow`: https://stackoverflow.com/questions/tagged/ray
.. _`Meetup Group`: https://www.meetup.com/Bay-Area-Ray-Meetup/
build-docker.sh
+2 -2
View File
@@ -7,7 +7,7 @@ set -x
GPU=""
BASE_IMAGE="ubuntu:focal"
WHEEL_URL="https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl"
WHEEL_URL="https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl"
PYTHON_VERSION=""
while [[ $# -gt 0 ]]
@@ -16,7 +16,7 @@ key="$1"
case $key in
--gpu)
GPU="-gpu"
BASE_IMAGE="nvidia/cuda:10.1-cudnn7d-runtime-ubuntu18.04"
BASE_IMAGE="nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04"
;;
--no-cache-build)
NO_CACHE="--no-cache"
ci/performance_tests/README.rst
-31
View File
@@ -1,31 +0,0 @@
Performance Tests
=================
This directory contains scripts for running performance benchmarks. These
benchmarks are intended to be used by Ray developers to check if a given pull
request introduces a performance regression.
To check if a pull request introduces a performance regression, it is necessary
to run these benchmarks on the codebase before and after the change.
Running the Workloads
---------------------
To run the workload on a single machine, do the following.
.. code-block:: bash
python test_performance.py --num-nodes=3
This will start simulate a 3 node cluster on your local machine, attach to it,
and run the benchmarks. To run the benchmarks on an existing cluster, do the
following.
.. code-block:: bash
python test_performance.py --num-nodes=3 --address=<redis-address>
The ``--num-nodes`` flag must match the number of nodes in the cluster. The
nodes in the cluster must be configured with the appropriate resource labels. In
particular, the ith node in the cluster must have a resource named ``"i"``
with quantity ``500``.
ci/performance_tests/test_performance.py
-248
View File
@@ -1,248 +0,0 @@
import argparse
import logging
import numpy as np
import time
import ray
from ray.tests.cluster_utils import Cluster
logger = logging.getLogger(__name__)
parser = argparse.ArgumentParser(
description="Parse arguments for running the performance tests.")
parser.add_argument(
"--num-nodes",
required=True,
type=int,
help="The number of nodes to simulate in the cluster.")
parser.add_argument(
"--skip-object-store-warmup",
default=False,
action="store_true",
help="True if the object store should not be warmed up. This could cause "
"the benchmarks to appear slower than usual.")
parser.add_argument(
"--address",
required=False,
type=str,
help="The address of the cluster to connect to. If this is ommitted, then "
"a cluster will be started locally (on a single machine).")
def start_local_cluster(num_nodes, object_store_memory):
"""Start a local Ray cluster.
The ith node in the cluster will have a resource named "i".
Args:
num_nodes: The number of nodes to start in the cluster.
Returns:
The cluster object.
"""
num_redis_shards = 2
redis_max_memory = 10**8
cluster = Cluster()
for i in range(num_nodes):
cluster.add_node(
redis_port=6379 if i == 0 else None,
num_redis_shards=num_redis_shards if i == 0 else None,
num_cpus=8 if i == 0 else 2,
num_gpus=0,
resources={str(i): 500},
object_store_memory=object_store_memory,
redis_max_memory=redis_max_memory)
ray.init(address=cluster.address)
return cluster
def wait_for_and_check_cluster_configuration(num_nodes):
"""Check that the cluster's custom resources are properly configured.
The ith node should have a resource labeled 'i' with quantity 500.
Args:
num_nodes: The number of nodes that we expect to be in the cluster.
Raises:
RuntimeError: This exception is raised if the cluster is not configured
properly for this test.
"""
logger.warning("Waiting for cluster to have %s nodes.", num_nodes)
while True:
nodes = ray.nodes()
if len(nodes) == num_nodes:
break
if len(nodes) > num_nodes:
raise RuntimeError(
"The cluster has %s nodes, but it should "
"only have %s.", len(nodes), num_nodes)
if not ([set(node["Resources"].keys())
for node in ray.nodes()] == [{str(i), "CPU"}
for i in range(num_nodes)]):
raise RuntimeError(
"The ith node in the cluster should have a "
"custom resource called 'i' with quantity "
"500. The nodes are\n%s", ray.nodes())
if not ([[
resource_quantity
for resource_name, resource_quantity in node["Resources"].items()
if resource_name != "CPU"
] for node in ray.nodes()] == num_nodes * [[500.0]]):
raise RuntimeError(
"The ith node in the cluster should have a "
"custom resource called 'i' with quantity "
"500. The nodes are\n%s", ray.nodes())
for node in ray.nodes():
if ("0" in node["Resources"] and node["ObjectStoreSocketName"] !=
ray.worker.global_worker.plasma_client.store_socket_name):
raise RuntimeError("The node that this driver is connected to "
"must have a custom resource labeled '0'.")
@ray.remote
def create_array(size):
return np.zeros(shape=size, dtype=np.uint8)
@ray.remote
def no_op(*values):
# The reason that this function takes *values is so that we can pass in
# an arbitrary number of object refs to create task dependencies.
return 1
@ray.remote
class Actor(object):
def ping(self, *values):
pass
def warm_up_cluster(num_nodes, object_store_memory):
"""Warm up the cluster.
This will allocate enough objects in each object store to cause eviction
because the first time a driver or worker touches a region of memory in the
object store, it may be slower.
Note that remote functions are exported lazily, so the first invocation of
a given remote function will be slower.
"""
logger.warning("Warming up the object store.")
size = object_store_memory * 2 // 5
num_objects = 2
while size > 0:
object_refs = []
for i in range(num_nodes):
for _ in range(num_objects):
object_refs += [
create_array._remote(args=[size], resources={str(i): 1})
]
size = size // 2
num_objects = min(num_objects * 2, 1000)
for object_ref in object_refs:
ray.get(object_ref)
logger.warning("Finished warming up the object store.")
# Invoke all of the remote functions once so that the definitions are
# broadcast to the workers.
ray.get(no_op.remote())
ray.get(Actor.remote().ping.remote())
def run_multiple_trials(f, num_trials):
durations = []
for _ in range(num_trials):
start = time.time()
f()
durations.append(time.time() - start)
return durations
def test_tasks(num_nodes):
def one_thousand_serial_tasks_local_node():
for _ in range(1000):
ray.get(no_op._remote(resources={"0": 1}))
durations = run_multiple_trials(one_thousand_serial_tasks_local_node, 10)
logger.warning(
"one_thousand_serial_tasks_local_node \n"
" min: %.2gs\n"
" mean: %.2gs\n"
" std: %.2gs", np.min(durations), np.mean(durations),
np.std(durations))
def one_thousand_serial_tasks_remote_node():
for _ in range(1000):
ray.get(no_op._remote(resources={"1": 1}))
durations = run_multiple_trials(one_thousand_serial_tasks_remote_node, 10)
logger.warning(
"one_thousand_serial_tasks_remote_node \n"
" min: %.2gs\n"
" mean: %.2gs\n"
" std: %.2gs", np.min(durations), np.mean(durations),
np.std(durations))
def ten_thousand_parallel_tasks_local():
ray.get([no_op._remote(resources={"0": 1}) for _ in range(10000)])
durations = run_multiple_trials(ten_thousand_parallel_tasks_local, 5)
logger.warning(
"ten_thousand_parallel_tasks_local \n"
" min: %.2gs\n"
" mean: %.2gs\n"
" std: %.2gs", np.min(durations), np.mean(durations),
np.std(durations))
def ten_thousand_parallel_tasks_load_balanced():
ray.get([
no_op._remote(resources={str(i % num_nodes): 1})
for i in range(10000)
])
durations = run_multiple_trials(ten_thousand_parallel_tasks_load_balanced,
5)
logger.warning(
"ten_thousand_parallel_tasks_load_balanced \n"
" min: %.2gs\n"
" mean: %.2gs\n"
" std: %.2gs", np.min(durations), np.mean(durations),
np.std(durations))
if __name__ == "__main__":
args = parser.parse_args()
num_nodes = args.num_nodes
object_store_memory = 10**8
# Configure the cluster or check that it is properly configured.
if num_nodes < 2:
raise ValueError("The --num-nodes argument must be at least 2.")
if args.address:
ray.init(address=args.address)
wait_for_and_check_cluster_configuration(num_nodes)
logger.warning(
"Running performance benchmarks on the cluster with "
"address %s.", args.address)
else:
logger.warning(
"Running performance benchmarks on a simulated cluster "
"of %s nodes.", num_nodes)
cluster = start_local_cluster(num_nodes, object_store_memory)
if not args.skip_object_store_warmup:
warm_up_cluster(num_nodes, object_store_memory)
# Run the benchmarks.
test_tasks(num_nodes)
# TODO(rkn): Test actors, test object transfers, test tasks with many
# dependencies.
ci/travis/format.sh
+6 -7
View File
@@ -97,7 +97,8 @@ MYPY_FILES=(
'autoscaler/node_provider.py'
'autoscaler/sdk.py'
'autoscaler/_private/commands.py'
'operator.py'
'operator/operator.py'
'operator/operator_utils.py'
)
YAPF_EXCLUDES=(
@@ -114,8 +115,6 @@ GIT_LS_EXCLUDES=(
# TODO(barakmich): This should be cleaned up. I've at least excised the copies
# of these arguments to this location, but the long-term answer is to actually
# make a flake8 config file
FLAKE8_EXCLUDE="--exclude=python/ray/core/generated/,streaming/python/generated,doc/source/conf.py,python/ray/cloudpickle/,python/ray/thirdparty_files/,python/build/,python/.eggs/"
FLAKE8_IGNORES="--ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605"
FLAKE8_PYX_IGNORES="--ignore=C408,E121,E123,E126,E211,E225,E226,E227,E24,E704,E999,W503,W504,W605"
shellcheck_scripts() {
@@ -195,10 +194,10 @@ format_all() {
if [ $HAS_FLAKE8 ]; then
echo "$(date)" "Flake8...."
git ls-files -- '*.py' "${GIT_LS_EXCLUDES[@]}" | xargs -P 5 \
flake8 --inline-quotes '"' --no-avoid-escape "$FLAKE8_EXCLUDE" "$FLAKE8_IGNORES"
flake8 --config=.flake8
git ls-files -- '*.pyx' '*.pxd' '*.pxi' "${GIT_LS_EXCLUDES[@]}" | xargs -P 5 \
flake8 --inline-quotes '"' --no-avoid-escape "$FLAKE8_EXCLUDE" "$FLAKE8_PYX_IGNORES"
flake8 --config=.flake8 "$FLAKE8_PYX_IGNORES"
fi
echo "$(date)" "clang-format...."
@@ -237,14 +236,14 @@ format_changed() {
yapf --in-place "${YAPF_EXCLUDES[@]}" "${YAPF_FLAGS[@]}"
if which flake8 >/dev/null; then
git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.py' | xargs -P 5 \
flake8 --inline-quotes '"' --no-avoid-escape "$FLAKE8_EXCLUDE" "$FLAKE8_IGNORES"
flake8 --config=.flake8
fi
fi
if ! git diff --diff-filter=ACRM --quiet --exit-code "$MERGEBASE" -- '*.pyx' '*.pxd' '*.pxi' &>/dev/null; then
if which flake8 >/dev/null; then
git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.pyx' '*.pxd' '*.pxi' | xargs -P 5 \
flake8 --inline-quotes '"' --no-avoid-escape "$FLAKE8_EXCLUDE" "$FLAKE8_PYX_IGNORES"
flake8 --config=.flake8 "$FLAKE8_PYX_IGNORES"
fi
fi
ci/travis/install-dependencies.sh
+4 -3
View File
@@ -254,7 +254,7 @@ install_dependencies() {
local torch_url="https://download.pytorch.org/whl/torch_stable.html"
case "${OSTYPE}" in
darwin*) pip install torch torchvision;;
*) pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f "${torch_url}";;
*) pip install torch==1.7.0+cpu torchvision==0.8.1+cpu -f "${torch_url}";;
esac
# Try n times; we often encounter OpenSSL.SSL.WantReadError (or others)
@@ -312,12 +312,13 @@ install_dependencies() {
# If CI has deemed that a different version of Tensorflow or Torch
# should be installed, then upgrade/downgrade to that specific version.
if [ -n "${TORCH_VERSION-}" ] || [ -n "${TFP_VERSION-}" ] || [ -n "${TF_VERSION-}" ]; then
case "${TORCH_VERSION-1.6}" in
case "${TORCH_VERSION-1.7}" in
1.7) TORCHVISION_VERSION=0.8.1;;
1.5) TORCHVISION_VERSION=0.6.0;;
*) TORCHVISION_VERSION=0.5.0;;
esac
pip install --use-deprecated=legacy-resolver --upgrade tensorflow-probability=="${TFP_VERSION-0.8}" \
torch=="${TORCH_VERSION-1.6}" torchvision=="${TORCHVISION_VERSION}" \
torch=="${TORCH_VERSION-1.7}" torchvision=="${TORCHVISION_VERSION}" \
tensorflow=="${TF_VERSION-2.2.0}" gym
fi
cpp/src/ray/runtime/task/local_mode_task_submitter.cc
+1 -1
View File
@@ -44,7 +44,7 @@ ObjectID LocalModeTaskSubmitter::Submit(InvocationSpec &invocation) {
local_mode_ray_tuntime_.GetCurrentTaskId(), 0,
local_mode_ray_tuntime_.GetCurrentTaskId(), address, 1,
required_resources, required_placement_resources,
std::make_pair(PlacementGroupID::Nil(), -1), true);
std::make_pair(PlacementGroupID::Nil(), -1), true, "");
if (invocation.task_type == TaskType::NORMAL_TASK) {
} else if (invocation.task_type == TaskType::ACTOR_CREATION_TASK) {
invocation.actor_id = local_mode_ray_tuntime_.GetNextActorID();
cpp/src/ray/runtime/task/native_task_submitter.cc
+1 -1
View File
@@ -27,7 +27,7 @@ ObjectID NativeTaskSubmitter::Submit(InvocationSpec &invocation) {
} else {
core_worker.SubmitTask(BuildRayFunction(invocation), invocation.args, TaskOptions(),
&return_ids, 1, std::make_pair(PlacementGroupID::Nil(), -1),
true);
true, "");
}
return return_ids[0];
}
cpp/src/ray/runtime/task/task_executor.cc
+1 -1
View File
@@ -27,7 +27,7 @@ Status TaskExecutor::ExecuteTask(
const std::unordered_map<std::string, double> &required_resources,
const std::vector<std::shared_ptr<RayObject>> &args_buffer,
const std::vector<ObjectID> &arg_reference_ids,
const std::vector<ObjectID> &return_ids,
const std::vector<ObjectID> &return_ids, const std::string &debugger_breakpoint,
std::vector<std::shared_ptr<RayObject>> *results) {
RAY_LOG(INFO) << "TaskExecutor::ExecuteTask";
RAY_CHECK(ray_function.GetLanguage() == Language::CPP);
cpp/src/ray/runtime/task/task_executor.h
+1 -1
View File
@@ -38,7 +38,7 @@ class TaskExecutor {
const std::unordered_map<std::string, double> &required_resources,
const std::vector<std::shared_ptr<RayObject>> &args,
const std::vector<ObjectID> &arg_reference_ids,
const std::vector<ObjectID> &return_ids,
const std::vector<ObjectID> &return_ids, const std::string &debugger_breakpoint,
std::vector<std::shared_ptr<RayObject>> *results);
virtual ~TaskExecutor(){};
dashboard/BUILD
+13 -1
View File
@@ -13,7 +13,19 @@ py_library(
py_test_run_all_subdirectory(
size = "small",
include = ["**/test*.py"],
exclude = ["modules/test/**"],
exclude = ["modules/test/**", "modules/stats_collector/tests/test_stats_collector.py", "tests/test_dashboard.py"],
extra_srcs = [],
tags = ["exclusive"],
)
py_test(
name = "test_stats_collector",
size = "medium",
srcs = ["modules/stats_collector/tests/test_stats_collector.py"],
)
py_test(
name = "test_dashboard",
size = "medium",
srcs = ["tests/test_dashboard.py"],
)
dashboard/agent.py
+19 -12
View File
@@ -62,7 +62,9 @@ class DashboardAgent(object):
self.object_store_name = object_store_name
self.raylet_name = raylet_name
self.node_id = os.environ["RAY_NODE_ID"]
assert self.node_id, "Empty node id (RAY_NODE_ID)."
self.ppid = int(os.environ["RAY_RAYLET_PID"])
assert self.ppid > 0
logger.info("Parent pid is %s", self.ppid)
self.server = aiogrpc.server(options=(("grpc.so_reuseport", 0), ))
self.grpc_port = self.server.add_insecure_port(
f"[::]:{self.dashboard_agent_port}")
@@ -89,17 +91,21 @@ class DashboardAgent(object):
async def run(self):
async def _check_parent():
"""Check if raylet is dead."""
curr_proc = psutil.Process()
while True:
parent = curr_proc.parent()
if parent is None or parent.pid == 1:
logger.error("raylet is dead, agent will die because "
"it fate-shares with raylet.")
sys.exit(0)
await asyncio.sleep(
dashboard_consts.
DASHBOARD_AGENT_CHECK_PARENT_INTERVAL_SECONDS)
"""Check if raylet is dead and fate-share if it is."""
try:
curr_proc = psutil.Process()
while True:
parent = curr_proc.parent()
if (parent is None or parent.pid == 1
or self.ppid != parent.pid):
logger.error("Raylet is dead, exiting.")
sys.exit(0)
await asyncio.sleep(
dashboard_consts.
DASHBOARD_AGENT_CHECK_PARENT_INTERVAL_SECONDS)
except Exception:
logger.error("Failed to check parent PID, exiting.")
sys.exit(1)
check_parent_task = create_task(_check_parent())
@@ -307,4 +313,5 @@ if __name__ == "__main__":
"error:\n{}".format(platform.uname()[1], traceback_str))
ray.utils.push_error_to_driver_through_redis(
redis_client, ray_constants.DASHBOARD_AGENT_DIED_ERROR, message)
logger.exception(message)
raise e
dashboard/client/package-lock.json
Generated
+18827 -1583
View File
File diff suppressed because it is too large Load Diff
dashboard/client/src/pages/dashboard/node-info/features/ObjectStoreMemory.tsx
+6 -4
View File
@@ -18,20 +18,22 @@ export const ClusterObjectStoreMemory: ClusterFeatureRenderFn = ({ nodes }) => {
nodes.map((n) => n.raylet.objectStoreAvailableMemory),
);
const totalUsed = sum(nodes.map((n) => n.raylet.objectStoreUsedMemory));
const total = totalUsed + totalAvailable;
return (
<div style={{ minWidth: 60 }}>
<UsageBar
percent={100 * (totalUsed / totalAvailable)}
text={formatUsage(totalUsed, totalAvailable, "mebibyte", false)}
percent={100 * (totalUsed / total)}
text={formatUsage(totalUsed, total, "mebibyte", false)}
/>
</div>
);
};
export const NodeObjectStoreMemory: NodeFeatureRenderFn = ({ node }) => {
const total = node.raylet.objectStoreAvailableMemory;
const totalAvailable = node.raylet.objectStoreAvailableMemory;
const used = node.raylet.objectStoreUsedMemory;
if (used === undefined || total === undefined || total === 0) {
const total = totalAvailable + used;
if (used === undefined || totalAvailable === undefined || total === 0) {
return (
<Typography color="textSecondary" component="span" variant="inherit">
N/A
dashboard/dashboard.py
+1
View File
@@ -207,4 +207,5 @@ if __name__ == "__main__":
if isinstance(e, OSError) and e.errno == errno.ENOENT:
logger.warning(message)
else:
logger.exception(message)
raise e
dashboard/datacenter.py
+2 -5
View File
@@ -238,7 +238,7 @@ class DataOrganizer:
pid = core_worker_stats.get("pid")
node_physical_stats = DataSource.node_physical_stats.get(node_id, {})
actor_process_stats = None
actor_process_gpu_stats = None
actor_process_gpu_stats = []
if pid:
for process_stats in node_physical_stats.get("workers", []):
if process_stats["pid"] == pid:
@@ -248,14 +248,11 @@ class DataOrganizer:
for gpu_stats in node_physical_stats.get("gpus", []):
for process in gpu_stats.get("processes", []):
if process["pid"] == pid:
actor_process_gpu_stats = gpu_stats
actor_process_gpu_stats.append(gpu_stats)
break
if actor_process_gpu_stats is not None:
break
actor["gpus"] = actor_process_gpu_stats
actor["processStats"] = actor_process_stats
return actor
@classmethod
dashboard/modules/logical_view/tests/test_logical_view_head.py
+7
View File
@@ -54,6 +54,13 @@ def test_actor_groups(ray_start_with_dashboard):
assert summary["stateToCount"]["ALIVE"] == 2
entries = actor_groups["Foo"]["entries"]
foo_entry = entries[0]
assert type(foo_entry["gpus"]) is list
assert "timestamp" in foo_entry
assert "actorConstructor" in foo_entry
assert "actorClass" in foo_entry
assert "actorId" in foo_entry
assert "ipAddress" in foo_entry
assert len(entries) == 2
assert "InfeasibleActor" in actor_groups
dashboard/modules/stats_collector/stats_collector_head.py
+10 -8
View File
@@ -147,20 +147,22 @@ class StatsCollector(dashboard_utils.DashboardHeadModule):
@routes.get("/node_logs")
async def get_logs(self, req) -> aiohttp.web.Response:
ip = req.query["ip"]
pid = req.query.get("pid")
node_logs = DataSource.ip_and_pid_to_logs[ip]
payload = node_logs.get(pid, []) if pid else node_logs
pid = str(req.query.get("pid", ""))
node_logs = DataSource.ip_and_pid_to_logs.get(ip, {})
if pid:
node_logs = {str(pid): node_logs.get(pid, [])}
return dashboard_utils.rest_response(
success=True, message="Fetched logs.", logs=payload)
success=True, message="Fetched logs.", logs=node_logs)
@routes.get("/node_errors")
async def get_errors(self, req) -> aiohttp.web.Response:
ip = req.query["ip"]
pid = req.query.get("pid")
node_errors = DataSource.ip_and_pid_to_errors[ip]
filtered_errs = node_errors.get(pid, []) if pid else node_errors
pid = str(req.query.get("pid", ""))
node_errors = DataSource.ip_and_pid_to_errors.get(ip, {})
if pid:
node_errors = {str(pid): node_errors.get(pid, [])}
return dashboard_utils.rest_response(
success=True, message="Fetched errors.", errors=filtered_errs)
success=True, message="Fetched errors.", errors=node_errors)
async def _update_actors(self):
# Subscribe actor channel.
dashboard/modules/stats_collector/tests/test_stats_collector.py
+166 -8
View File
@@ -4,15 +4,16 @@ import logging
import requests
import time
import traceback
import random
import pytest
import ray
import threading
from datetime import datetime, timedelta
from ray.cluster_utils import Cluster
from ray.new_dashboard.tests.conftest import * # noqa
from ray.test_utils import (
format_web_url,
wait_until_server_available,
wait_for_condition,
)
from ray.test_utils import (format_web_url, wait_until_server_available,
wait_for_condition,
wait_until_succeeded_without_exception)
logger = logging.getLogger(__name__)
@@ -184,7 +185,7 @@ def test_get_all_node_details(disable_aiohttp_cache, ray_start_with_dashboard):
}], indirect=True)
def test_multi_nodes_info(enable_test_module, disable_aiohttp_cache,
ray_start_cluster_head):
cluster = ray_start_cluster_head
cluster: Cluster = ray_start_cluster_head
assert (wait_until_server_available(cluster.webui_url) is True)
webui_url = cluster.webui_url
webui_url = format_web_url(webui_url)
@@ -216,7 +217,164 @@ def test_multi_nodes_info(enable_test_module, disable_aiohttp_cache,
logger.info(ex)
return False
wait_for_condition(_check_nodes, timeout=10)
wait_for_condition(_check_nodes, timeout=15)
@pytest.mark.parametrize(
"ray_start_cluster_head", [{
"include_dashboard": True
}], indirect=True)
def test_multi_node_churn(enable_test_module, disable_aiohttp_cache,
ray_start_cluster_head):
cluster: Cluster = ray_start_cluster_head
assert (wait_until_server_available(cluster.webui_url) is True)
webui_url = format_web_url(cluster.webui_url)
def cluster_chaos_monkey():
worker_nodes = []
while True:
time.sleep(5)
if len(worker_nodes) < 2:
worker_nodes.append(cluster.add_node())
continue
should_add_node = random.randint(0, 1)
if should_add_node:
worker_nodes.append(cluster.add_node())
else:
node_index = random.randrange(0, len(worker_nodes))
node_to_remove = worker_nodes.pop(node_index)
cluster.remove_node(node_to_remove)
def get_index():
resp = requests.get(webui_url)
resp.raise_for_status()
def get_nodes():
resp = requests.get(webui_url + "/nodes?view=summary")
resp.raise_for_status()
summary = resp.json()
assert summary["result"] is True, summary["msg"]
assert summary["data"]["summary"]
t = threading.Thread(target=cluster_chaos_monkey, daemon=True)
t.start()
t_st = datetime.now()
duration = timedelta(seconds=60)
while datetime.now() < t_st + duration:
get_index()
time.sleep(2)
@pytest.mark.parametrize(
"ray_start_cluster_head", [{
"include_dashboard": True
}], indirect=True)
def test_logs(enable_test_module, disable_aiohttp_cache,
ray_start_cluster_head):
cluster = ray_start_cluster_head
assert (wait_until_server_available(cluster.webui_url) is True)
webui_url = cluster.webui_url
webui_url = format_web_url(webui_url)
nodes = ray.nodes()
assert len(nodes) == 1
node_ip = nodes[0]["NodeManagerAddress"]
@ray.remote
class LoggingActor:
def go(self, n):
i = 0
while i < n:
print(f"On number {i}")
i += 1
def get_pid(self):
return os.getpid()
la = LoggingActor.remote()
la2 = LoggingActor.remote()
la_pid = str(ray.get(la.get_pid.remote()))
la2_pid = str(ray.get(la2.get_pid.remote()))
ray.get(la.go.remote(4))
ray.get(la2.go.remote(1))
def check_logs():
node_logs_response = requests.get(
f"{webui_url}/node_logs", params={"ip": node_ip})
node_logs_response.raise_for_status()
node_logs = node_logs_response.json()
assert node_logs["result"]
assert type(node_logs["data"]["logs"]) is dict
assert all(
pid in node_logs["data"]["logs"] for pid in (la_pid, la2_pid))
assert len(node_logs["data"]["logs"][la2_pid]) == 1
actor_one_logs_response = requests.get(
f"{webui_url}/node_logs",
params={
"ip": node_ip,
"pid": str(la_pid)
})
actor_one_logs_response.raise_for_status()
actor_one_logs = actor_one_logs_response.json()
assert actor_one_logs["result"]
assert type(actor_one_logs["data"]["logs"]) is dict
assert len(actor_one_logs["data"]["logs"][la_pid]) == 4
wait_until_succeeded_without_exception(
check_logs, (AssertionError), timeout_ms=1000)
@pytest.mark.parametrize(
"ray_start_cluster_head", [{
"include_dashboard": True
}], indirect=True)
def test_errors(enable_test_module, disable_aiohttp_cache,
ray_start_cluster_head):
cluster = ray_start_cluster_head
assert (wait_until_server_available(cluster.webui_url) is True)
webui_url = cluster.webui_url
webui_url = format_web_url(webui_url)
nodes = ray.nodes()
assert len(nodes) == 1
node_ip = nodes[0]["NodeManagerAddress"]
@ray.remote
class ErrorActor():
def go(self):
raise ValueError("This is an error")
def get_pid(self):
return os.getpid()
ea = ErrorActor.remote()
ea_pid = ea.get_pid.remote()
ea.go.remote()
def check_errs():
node_errs_response = requests.get(
f"{webui_url}/node_logs", params={"ip": node_ip})
node_errs_response.raise_for_status()
node_errs = node_errs_response.json()
assert node_errs["result"]
assert type(node_errs["data"]["errors"]) is dict
assert ea_pid in node_errs["data"]["errors"]
assert len(node_errs["data"]["errors"][ea_pid]) == 1
actor_err_response = requests.get(
f"{webui_url}/node_logs",
params={
"ip": node_ip,
"pid": str(ea_pid)
})
actor_err_response.raise_for_status()
actor_errs = actor_err_response.json()
assert actor_errs["result"]
assert type(actor_errs["data"]["errors"]) is dict
assert len(actor_errs["data"]["errors"][ea_pid]) == 4
wait_until_succeeded_without_exception(
check_errs, (AssertionError), timeout_ms=1000)
if __name__ == "__main__":
dashboard/tests/test_dashboard.py
+6 -1
View File
@@ -80,7 +80,7 @@ def test_basic(ray_start_with_dashboard):
0]
dashboard_proc = psutil.Process(dashboard_proc_info.process.pid)
assert dashboard_proc.status() in [
psutil.STATUS_RUNNING, psutil.STATUS_SLEEPING
psutil.STATUS_RUNNING, psutil.STATUS_SLEEPING, psutil.STATUS_DISK_SLEEP
]
raylet_proc_info = all_processes[ray_constants.PROCESS_TYPE_RAYLET][0]
raylet_proc = psutil.Process(raylet_proc_info.process.pid)
@@ -137,6 +137,11 @@ def test_basic(ray_start_with_dashboard):
assert agent_proc.pid == agent_pid
time.sleep(1)
# The agent should be dead if raylet exits.
raylet_proc.kill()
raylet_proc.wait()
agent_proc.wait(5)
# Check redis keys are set.
logger.info("Check redis keys are set.")
dashboard_address = client.get(dashboard_consts.REDIS_KEY_DASHBOARD)
doc/examples/cython/ray-project/requirements.txt
+1 -1
View File
@@ -1,2 +1,2 @@
ray[debug]
ray
scipy
doc/examples/lbfgs/ray-project/requirements.txt
+1 -1
View File
@@ -1 +1 @@
ray[debug,rllib]
ray[rllib]
doc/examples/newsreader/ray-project/cluster.yaml
+1 -1
View File
@@ -24,7 +24,7 @@ setup_commands:
- echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc
- sudo apt -y update
- sudo apt -y install npm
- pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
- pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
# How Ray will authenticate with newly launched nodes.
auth:
doc/examples/newsreader/ray-project/requirements.txt
+1 -1
View File
@@ -1,3 +1,3 @@
ray[debug]
ray
atoma
flask
doc/examples/placement-group.rst
+2 -1
View File
@@ -28,9 +28,10 @@ Gang Scheduling
---------------
**Recommended Strategy**: `STRICT_SPREAD`.
Sometimes, you'd like to schedule multiple tasks/actors in a separate physical machine (node) "at the same time". For example, "write the gang scheduling example".
Sometimes, you'd like to schedule multiple tasks/actors in a separate physical machine (node) "at the same time". For example, you may use a collective communication library like NCCL and form a communication group with a set of actors. In creating this set of actors, you may want to ensure that all actors are separated and placed evenly across available nodes (creating a homogenous cluster).
You can use placement groups' `STRICT_SPREAD` strategy to achieve it. `STRICT_SPREAD` ensures that all actors and tasks scheduled with the placement group will be located in a separate node.
Also, since the placement group creation is atomic, you can always guarantee that tasks and actors are scheduled at the same time.
Improve Fault tolerance
-----------------------
doc/examples/streaming/ray-project/requirements.txt
+1 -1
View File
@@ -1,2 +1,2 @@
ray[debug]
ray
wikipedia
doc/requirements-doc.txt
+1
View File
@@ -25,6 +25,7 @@ sphinx-jsonschema
sphinx-tabs
sphinx-version-warning
sphinx-book-theme
starlette
tabulate
uvicorn
werkzeug
doc/source/cluster/autoscaling.rst
+2 -2
View File
@@ -10,7 +10,7 @@ Basics
The Ray Cluster Launcher will automatically enable a load-based autoscaler. The scheduler will look at the task, actor, and placement group resource demands from the cluster, and tries to add the minimum set of nodes that can fulfill these demands. When nodes are idle for more than a timeout, they will be removed, down to the ``min_workers`` limit. The head node is never removed.
To avoid launching too many nodes at once, the number of nodes allowed to be pending is limited by the ``upscaling_speed`` setting. By default it is set to ``1.0``, which means the cluster can grow in size by at most ``100%`` at a time (doubling in size each time). This fraction can be set to as high as needed, e.g., ``99999`` to allow the cluster to quickly grow to its max size.
To avoid launching too many nodes at once, the number of nodes allowed to be pending is limited by the ``upscaling_speed`` setting. By default it is set to ``1.0``, which means the cluster can be growing in size by at most ``100%`` at any time (e.g., if the cluster currently has 20 nodes, at most 20 pending launches are allowed). This fraction can be set to as high as needed, e.g., ``99999`` to allow the cluster to quickly grow to its max size.
In more detail, the autoscaler implements the following control loop:
@@ -124,7 +124,7 @@ The node config tells the underlying Cloud provider how to launch a node of this
node_config:
InstanceType: p2.xlarge
The resources field tells the autoscaler what kinds of resources this node provides. This can include custom resources as well (e.g., "Custom2"). This field enables the autoscaler to automatically select the right kind of nodes to launch given the resource demands of the application. The resources specified here will be automatically passed to the ``ray start`` command for the node via an environment variable. For more information, see also the `resource demand scheduler <https://github.com/ray-project/ray/blob/master/python/ray/autoscaler/resource_demand_scheduler.py>`__:
The resources field tells the autoscaler what kinds of resources this node provides. This can include custom resources as well (e.g., "Custom2"). This field enables the autoscaler to automatically select the right kind of nodes to launch given the resource demands of the application. The resources specified here will be automatically passed to the ``ray start`` command for the node via an environment variable. For more information, see also the `resource demand scheduler <https://github.com/ray-project/ray/blob/master/python/ray/autoscaler/_private/resource_demand_scheduler.py>`__:
.. code-block:: yaml
doc/source/cluster/index.rst
+20 -14
View File
@@ -21,26 +21,34 @@ Clusters are started with the :ref:`Ray Cluster Launcher <ref-automatic-cluster>
You can also create a Ray cluster using a standard cluster manager such as :ref:`Kubernetes <ray-k8s-deploy>`, :ref:`YARN <ray-yarn-deploy>`, or :ref:`SLURM <ray-slurm-deploy>`.
After a cluster is started, you need to connect your program to the Ray cluster.
After a cluster is started, you need to connect your program to the Ray cluster by starting a driver process on the same node as where you ran ``ray start``:
.. tabs::
.. group-tab:: python
.. code-tab:: python
You can connect to this Ray runtime by starting a Python process that calls the following on the same node as where you ran ``ray start``:
.. code-block:: python
# This must
import ray
ray.init(address='auto')
# This must
import ray
ray.init(address='auto')
.. group-tab:: java
If you want to run Java code, you need to specify the classpath via the ``--code-search-path`` option. See :ref:`code_search_path` for more details.
.. code-block:: java
import io.ray.api.Ray;
public class MyRayApp {
public static void main(String[] args) {
Ray.init();
...
}
}
.. code-block:: bash
$ ray start ... --code-search-path=/path/to/jars
java -classpath <classpath> \
-Dray.address=<address> \
<classname> <args>
and then the rest of your script should be able to leverage Ray as a distributed framework!
@@ -74,8 +82,6 @@ The most preferable way to run a Ray cluster is via the :ref:`Ray Cluster Launch
This section assumes that you have a list of machines and that the nodes in the cluster can communicate with each other. It also assumes that Ray is installed
on each machine. To install Ray, follow the `installation instructions`_.
To configure the Ray cluster to run Java code, you need to add the ``--code-search-path`` option. See :ref:`code_search_path` for more details.
.. _`installation instructions`: http://docs.ray.io/en/master/installation.html
Starting Ray on each machine
@@ -199,7 +205,7 @@ To run a distributed Ray program, you'll need to execute your program on the sam
.. code-block:: bash
java -classpath /path/to/jars/ \
java -classpath <classpath> \
-Dray.address=<address> \
<classname> <args>
doc/source/cluster/k8s-operator.rst
+234
View File
@@ -0,0 +1,234 @@
.. _k8s-operator:
The Ray Kubernetes Operator
=================================
Ray provides a `Kubernetes Operator`_ for managing autoscaling Ray clusters.
Using the operator provides similar functionality to deploying a Ray cluster using
the :ref:`Ray Cluster Launcher<ref-autoscaling>`. However, working with the operator does not require
running Ray locally -- all interactions with your Ray cluster are mediated by Kubernetes.
The operator makes use of a `Kubernetes Custom Resource`_ called a *RayCluster*.
A RayCluster is specified by a configuration similar to the ``yaml`` files used by the Ray Cluster Launcher.
Internally, the operator uses Ray's autoscaler to manage your Ray cluster. However, the autoscaler runs in a
separate operator pod, rather than on the Ray head node. Applying multiple RayCluster custom resources in the operator's
namespace allows the operator to manage several Ray clusters.
The rest of this document explains step-by-step how to use the Ray Kubernetes Operator to launch a Ray cluster on your existing Kubernetes cluster.
.. role:: bash(code)
:language: bash
.. note::
The example commands in this document launch six Kubernetes pods, using a total of 6 CPU and 3.5Gi memory.
If you are experimenting using a test Kubernetes environment such as `minikube`_, make sure to provision sufficient resources, e.g.
:bash:`minikube start --cpus=6 --memory=\"4G\"`.
Alternatively, reduce resource usage by editing the ``yaml`` files referenced in this document; for example, reduce ``minWorkers``
in ``example_cluster.yaml`` and ``example_cluster2.yaml``.
Applying the RayCluster Custom Resource Definition
--------------------------------------------------
First, we need to apply the `Kubernetes Custom Resource Definition`_ (CRD) defining a RayCluster.
.. note::
Creating a Custom Resource Definition requires the appropriate Kubernetes cluster-level privileges.
.. code-block:: shell
$ kubectl apply -f ray/python/ray/autoscaler/kubernetes/operator_configs/cluster_crd.yaml
customresourcedefinition.apiextensions.k8s.io/rayclusters.cluster.ray.io created
Picking a Kubernetes Namespace
-------------------------------
The rest of the Kubernetes resources we will use are `namespaced`_.
You can use an existing namespace for your Ray clusters or create a new one if you have permissions.
For this example, we will create a namespace called ``ray``.
.. code-block:: shell
$ kubectl create namespace ray
namespace/ray created
Starting the Operator
----------------------
To launch the operator in our namespace, we execute the following command.
.. code-block:: shell
$ kubectl -n ray apply -f ray/python/ray/autoscaler/kubernetes/operator_configs/operator.yaml
serviceaccount/ray-operator-serviceaccount created
role.rbac.authorization.k8s.io/ray-operator-role created
rolebinding.rbac.authorization.k8s.io/ray-operator-rolebinding created
pod/ray-operator-pod created
The output shows that we've launched a Pod named ``ray-operator-pod``. This is the pod that runs the operator process.
The ServiceAccount, Role, and RoleBinding we have created grant the operator pod the `permissions`_ it needs to manage Ray clusters.
Launching Ray Clusters
----------------------
Finally, to launch a Ray cluster, we create a RayCluster custom resource.
.. code-block:: shell
$ kubectl -n ray apply -f ray/python/ray/autoscaler/kubernetes/operator_configs/example_cluster.yaml
raycluster.cluster.ray.io/example-cluster created
The operator detects the RayCluster resource we've created and launches an autoscaling Ray cluster.
Our RayCluster configuration specifies ``minWorkers:2`` in the second entry of ``spec.podTypes``, so we get a head node and two workers upon launch.
.. note::
For more details about RayCluster resources, we recommend take a looking at the annotated example ``example_cluster.yaml`` applied in the last command.
.. code-block:: shell
$ kubectl -n ray get pods
NAME READY STATUS RESTARTS AGE
example-cluster-ray-head-hbxvv 1/1 Running 0 72s
example-cluster-ray-worker-4hvv6 1/1 Running 0 64s
example-cluster-ray-worker-78kp5 1/1 Running 0 64s
ray-operator-pod 1/1 Running 0 2m33s
We see four pods: the operator, the Ray head node, and two Ray worker nodes.
Let's launch another cluster in the same namespace, this one specifiying ``minWorkers:1``.
.. code-block:: shell
$ kubectl -n ray apply -f ray/python/ray/autoscaler/kubernetes/operator_configs/example_cluster2.yaml
We confirm that both clusters are running in our namespace.
.. code-block:: shell
$ kubectl -n ray get rayclusters
NAME AGE
example-cluster 12m
example-cluster2 114s
$ kubectl -n ray get pods
NAME READY STATUS RESTARTS AGE
example-cluster-ray-head-th4wv 1/1 Running 0 10m
example-cluster-ray-worker-q9pjn 1/1 Running 0 10m
example-cluster-ray-worker-qltnp 1/1 Running 0 10m
example-cluster2-ray-head-kj5mg 1/1 Running 0 10s
example-cluster2-ray-worker-qsgnd 1/1 Running 0 1s
ray-operator-pod 1/1 Running 0 10m
Now we can :ref:`run Ray programs<ray-k8s-run>` on our Ray clusters.
Monitoring
----------
Autoscaling logs are written to the operator pod's ``stdout`` and can be accessed with :code:`kubectl logs`.
Each line of output is prefixed by the name of the cluster followed by a colon.
The following command gets the last hundred lines of autoscaling logs for our second cluster.
.. code-block:: shell
$ kubectl -n ray logs ray-operator-pod | grep ^example-cluster2: | tail -n 100
The output should include monitoring updates that look like this:
.. code-block:: shell
example-cluster2:2020-12-12 13:55:36,814 DEBUG autoscaler.py:693 -- Cluster status: 1 nodes
example-cluster2: - MostDelayedHeartbeats: {'172.17.0.4': 0.04093289375305176, '172.17.0.5': 0.04084634780883789}
example-cluster2: - NodeIdleSeconds: Min=36 Mean=38 Max=41
example-cluster2: - ResourceUsage: 0.0/2.0 CPU, 0.0/1.0 Custom1, 0.0/1.0 is_spot, 0.0 GiB/0.58 GiB memory, 0.0 GiB/0.1 GiB object_store_memory
example-cluster2: - TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
example-cluster2:Worker node types:
example-cluster2: - worker-nodes: 1
example-cluster2:2020-12-12 13:55:36,870 INFO resource_demand_scheduler.py:148 -- Cluster resources: [{'object_store_memory': 1.0, 'node:172.17.0.4': 1.0, 'memory': 5.0, 'CPU': 1.0}, {'object_store_memory': 1.0, 'is_spot': 1.0, 'memory': 6.0, 'node:172.17.0.5': 1.0, 'Custom1': 1.0, 'CPU': 1.0}]
example-cluster2:2020-12-12 13:55:36,870 INFO resource_demand_scheduler.py:149 -- Node counts: defaultdict(<class 'int'>, {'head-node': 1, 'worker-nodes
': 1})
example-cluster2:2020-12-12 13:55:36,870 INFO resource_demand_scheduler.py:159 -- Placement group demands: []
example-cluster2:2020-12-12 13:55:36,870 INFO resource_demand_scheduler.py:186 -- Resource demands: []
example-cluster2:2020-12-12 13:55:36,870 INFO resource_demand_scheduler.py:187 -- Unfulfilled demands: []
example-cluster2:2020-12-12 13:55:36,891 INFO resource_demand_scheduler.py:209 -- Node requests: {}
example-cluster2:2020-12-12 13:55:36,903 DEBUG autoscaler.py:654 -- example-cluster2-ray-worker-tdxdr is not being updated and passes config check (can_update=True).
example-cluster2:2020-12-12 13:55:36,923 DEBUG autoscaler.py:654 -- example-cluster2-ray-worker-tdxdr is not being updated and passes config check (can_update=True).
Updating and Retrying
---------------------
To update a Ray cluster's configuration, edit the ``yaml`` file of the corresponding RayCluster resource
and apply it again:
.. code-block:: shell
$ kubectl -n ray apply -f ray/python/ray/autoscaler/kubernetes/operator_configs/example_cluster.yaml
To force a restart with the same configuration, you can add an `annotation`_ to the RayCluster resource's ``metadata.labels`` field, e.g.
.. code-block:: yaml
apiVersion: cluster.ray.io/v1
kind: RayCluster
metadata:
name: example-cluster
annotations:
try: again
spec:
...
Then reapply the RayCluster, as above.
Currently, editing and reapplying a RayCluster resource will stop and restart Ray processes running on the corresponding
Ray cluster. Similarly, deleting and relaunching the operator pod will stop and restart Ray processes on all Ray clusters in the operator's namespace.
This behavior may be modified in future releases.
Cleaning Up
-----------
We shut down a Ray cluster by deleting the associated RayCluster resource.
Either of the next two commands will delete our second cluster ``example-cluster2``.
.. code-block:: shell
$ kubectl -n ray delete raycluster example-cluster2
# OR
$ kubectl -n ray delete -f ray/python/ray/autoscaler/kubernetes/operator_configs/example_cluster2.yaml
The pods associated with ``example-cluster2`` go into ``TERMINATING`` status. In a few moments, we check that these pods are gone:
.. code-block:: shell
$ kubectl -n ray get pods
NAME READY STATUS RESTARTS AGE
example-cluster-ray-head-th4wv 1/1 Running 0 57m
example-cluster-ray-worker-q9pjn 1/1 Running 0 56m
example-cluster-ray-worker-qltnp 1/1 Running 0 56m
ray-operator-pod 1/1 Running 0 57m
Only the operator pod and the first ``example-cluster`` remain.
To finish clean-up, we delete the cluster ``example-cluster`` and then the operator's resources.
.. code-block:: shell
$ kubectl -n ray delete raycluster example-cluster
$ kubectl -n ray delete -f ray/python/ray/autoscaler/kubernetes/operator_configs/operator.yaml
If you like, you can delete the RayCluster customer resource definition.
(Using the operator again will then require reapplying the CRD.)
.. code-block:: shell
$ kubectl delete crd rayclusters.cluster.ray.io
# OR
$ kubectl delete -f ray/python/ray/autoscaler/kubernetes/operator_configs/cluster_crd.yaml
.. _`Kubernetes Operator`: https://kubernetes.io/docs/concepts/extend-kubernetes/operator/
.. _`Kubernetes Custom Resource`: https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/
.. _`Kubernetes Custom Resource Definition`: https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/
.. _`annotation`: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#attaching-metadata-to-objects
.. _`permissions`: https://kubernetes.io/docs/reference/access-authn-authz/rbac/
.. _`minikube`: https://minikube.sigs.k8s.io/docs/start/
.. _`namespaced`: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/
doc/source/cluster/kubernetes.rst
+10
View File
@@ -12,6 +12,9 @@ This document assumes that you have access to a Kubernetes cluster and have
first walk you through how to deploy a Ray cluster on your existing Kubernetes
cluster, then explore a few different ways to run programs on the Ray cluster.
To learn about deploying an autoscaling Ray cluster using :ref:`Ray's Kubernetes operator<k8s-operator>`, read
:ref:`here<k8s-operator>`.
The configuration ``yaml`` files used here are provided in the `Ray repository`_
as examples to get you started. When deploying real applications, you will probably
want to build and use your own container images, add more worker nodes to the
@@ -38,6 +41,11 @@ flag passed to ``kubectl``.
Starting a Ray Cluster
----------------------
.. toctree::
:hidden:
/cluster/k8s-operator.rst
A Ray cluster consists of a single head node and a set of worker nodes (the
provided ``ray-cluster.yaml`` file will start 3 worker nodes). In the example
Kubernetes configuration, this is implemented as:
@@ -142,6 +150,8 @@ and checking that they are restarted by Kubernetes:
ray-worker-5c49b7cc57-6m4kp 1/1 Running 0 10s
ray-worker-5c49b7cc57-jx2w2 1/1 Running 0 10s
.. _ray-k8s-run:
Running Ray Programs
--------------------
doc/source/conf.py
+3
View File
@@ -94,6 +94,9 @@ for mod_name in MOCK_MODULES:
sys.modules["tensorflow"].VERSION = "9.9.9"
sys.modules["tensorflow.keras.callbacks"] = ChildClassMock()
sys.modules["pytorch_lightning"] = ChildClassMock()
sys.modules["xgboost"] = ChildClassMock()
sys.modules["xgboost.core"] = ChildClassMock()
sys.modules["xgboost.callback"] = ChildClassMock()
class SimpleClass(object):
doc/source/configure.rst
+24 -7
View File
@@ -60,7 +60,9 @@ If using the command line, connect to the Ray cluster as follow:
override this by explicitly setting ``OMP_NUM_THREADS``. ``OMP_NUM_THREADS`` is commonly
used in numpy, PyTorch, and Tensorflow to perform multit-threaded linear algebra.
In multi-worker setting, we want one thread per worker instead of many threads
per worker to avoid contention.
per worker to avoid contention. Some other libraries may have their own way to configure
parallelism. For example, if you're using OpenCV, you should manually set the number of
threads using cv2.setNumThreads(num_threads) (set to 0 to disable multi-threading).
.. _temp-dir-log-files:
@@ -243,24 +245,32 @@ Java Applications
Code Search Path
~~~~~~~~~~~~~~~~
If you want to run a Java application in cluster mode, you must first run ``ray start`` to start the Ray cluster. In addition to any ``ray start`` parameters mentioned above, you must add ``--code-search-path`` to tell Ray where to load jars when starting Java workers. Your jar files must be distributed to all nodes of the Ray cluster before running your code, and this parameter must be set on both the head node and non-head nodes.
If you want to run a Java application in a multi-node cluster, you must specify the code search path in your driver. The code search path is to tell Ray where to load jars when starting Java workers. Your jar files must be distributed to the same path(s) on all nodes of the Ray cluster before running your code.
.. code-block:: bash
$ ray start ... --code-search-path=/path/to/jars
$ java -classpath <classpath> \
-Dray.address=<address> \
-Dray.job.code-search-path=/path/to/jars/ \
<classname> <args>
The ``/path/to/jars`` here points to a directory which contains jars. All jars in the directory will be loaded by workers. You can also provide multiple directories for this parameter.
The ``/path/to/jars/`` here points to a directory which contains jars. All jars in the directory will be loaded by workers. You can also provide multiple directories for this parameter.
.. code-block:: bash
$ ray start ... --code-search-path=/path/to/jars1:/path/to/jars2:/path/to/pys1:/path/to/pys2
$ java -classpath <classpath> \
-Dray.address=<address> \
-Dray.job.code-search-path=/path/to/jars1:/path/to/jars2:/path/to/pys1:/path/to/pys2 \
<classname> <args>
Code search path is also used for loading Python code if it's specified. This is required for :ref:`cross_language`. If code search path is specified, you can only run Python remote functions which can be found in the code search path.
You don't need to configure code search path if you run a Java application in a single-node cluster.
You don't need to configure code search path if you run a Java application in single machine mode.
See ``ray.job.code-search-path`` under :ref:`Driver Options <java-driver-options>` for more information.
.. note:: Currently we don't provide a way to configure Ray when running a Java application in single machine mode. If you need to configure Ray, run ``ray start`` to start the Ray cluster first.
.. _java-driver-options:
Driver Options
~~~~~~~~~~~~~~
@@ -287,4 +297,11 @@ The list of available driver options:
- Type: ``Boolean``
- Default: ``false``
- ``ray.job.code-search-path``
- The paths for Java workers to load code from. Currently only directories are supported. You can specify one or more directories split by a ``:``. You don't need to configure code search path if you run a Java application in single machine mode or local mode. Code search path is also used for loading Python code if it's specified. This is required for :ref:`cross_language`. If code search path is specified, you can only run Python remote functions which can be found in the code search path.
- Type: ``String``
- Default: empty string.
- Example: ``/path/to/jars1:/path/to/jars2:/path/to/pys1:/path/to/pys2``
.. _`Apache Arrow`: https://arrow.apache.org/
doc/source/cross-language.rst
+32 -6
View File
@@ -5,20 +5,46 @@ Cross-language programming
This page will show you how to use Ray's cross-language programming feature.
Setup the cluster
Setup the driver
-----------------
We need to set the ``--code-search-path`` option on ``ray start`` command. See :ref:`code_search_path` for more details.
We need to set :ref:`code_search_path` in your driver.
.. code-block:: bash
.. tabs::
ray start ... --code-search-path=/path/to/code
.. group-tab:: Python
.. code-block:: python
ray.init(job_config=ray.job_config.JobConfig(code_search_path="/path/to/code"))
.. group-tab:: Java
.. code-block:: bash
java -classpath <classpath> \
-Dray.address=<address> \
-Dray.job.code-search-path=/path/to/code/ \
<classname> <args>
You may want to include multiple directories to load both Python and Java code for workers, if they are placed in different directories.
.. code-block:: bash
.. tabs::
ray start ... --code-search-path=/path/to/jars:/path/to/pys
.. group-tab:: Python
.. code-block:: python
ray.init(job_config=ray.job_config.JobConfig(code_search_path="/path/to/jars:/path/to/pys"))
.. group-tab:: Java
.. code-block:: bash
java -classpath <classpath> \
-Dray.address=<address> \
-Dray.job.code-search-path=/path/to/jars:/path/to/pys \
<classname> <args>
Python calling Java
-------------------
doc/source/development.rst
+1 -1
View File
@@ -23,7 +23,7 @@ RLlib, Tune, Autoscaler, and most Python files do not require you to build and c
.. code-block:: shell
pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-manylinux2014_x86_64.whl
pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp38-cp38-manylinux2014_x86_64.whl
2. Fork and clone the project to your machine. Connect your repository to the upstream (main project) ray repository.
doc/source/getting-involved.rst
+5 -2
View File
@@ -6,7 +6,10 @@ Getting Involved / Contributing
Ray is more than a framework for distributed applications but also an active community of developers,
researchers, and folks that love machine learning.
.. tip:: Join our `community slack <https://forms.gle/9TSdDYUgxYs8SA9e8>`_ to discuss Ray! The community is extremely active in helping people succeed in building their ray applications.
.. tip:: Join our `community Slack <https://forms.gle/9TSdDYUgxYs8SA9e8>`_ to
discuss Ray or ask questions on `our forum <https://discuss.ray.io/>`_! The
community is extremely active in helping people succeed in building their
Ray applications.
You can join (and Star!) us on `on GitHub`_.
@@ -141,7 +144,7 @@ You can run the following locally:
.. code-block:: shell
ray/scripts/format.sh
./ci/travis/format.sh
An output like the following indicates failure:
doc/source/installation.rst
+82 -20
View File
@@ -15,7 +15,7 @@ You can install the latest official version of Ray as follows. Official releases
.. code-block:: bash
pip install -U ray # also recommended: ray[debug]
pip install -U ray
**Note for Windows Users:** To use Ray on Windows, Visual C++ runtime must be installed (see :ref:`Windows Dependencies <windows-dependencies>` section). If you run into any issues, please see the :ref:`Windows Support <windows-support>` section.
@@ -55,20 +55,21 @@ instead of the ones above:
`Linux Python 3.6`_ `MacOS Python 3.6`_ `Windows Python 3.6`_
=================== =================== ======================
.. _`Linux Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp39-cp39-manylinux2014_x86_64.whl
.. _`Linux Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-manylinux2014_x86_64.whl
.. _`Linux Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
.. _`Linux Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
.. _`MacOS Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp39-cp39-macosx_10_13_x86_64.whl
.. _`MacOS Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-macosx_10_13_x86_64.whl
.. _`MacOS Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-macosx_10_13_intel.whl
.. _`MacOS Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-macosx_10_13_intel.whl
.. _`Linux Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp39-cp39-manylinux2014_x86_64.whl
.. _`Linux Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp38-cp38-manylinux2014_x86_64.whl
.. _`Linux Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
.. _`Linux Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
.. _`Windows Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp39-cp39-win_amd64.whl
.. _`Windows Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-win_amd64.whl
.. _`Windows Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-win_amd64.whl
.. _`Windows Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-win_amd64.whl
.. _`MacOS Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp39-cp39-macosx_10_13_x86_64.whl
.. _`MacOS Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp38-cp38-macosx_10_13_x86_64.whl
.. _`MacOS Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-macosx_10_13_intel.whl
.. _`MacOS Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-macosx_10_13_intel.whl
.. _`Windows Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp39-cp39-win_amd64.whl
.. _`Windows Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp38-cp38-win_amd64.whl
.. _`Windows Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-win_amd64.whl
.. _`Windows Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-win_amd64.whl
Installing from a specific commit
@@ -80,11 +81,11 @@ You can install the Ray wheels of any particular commit on ``master`` with the f
pip install https://ray-wheels.s3-us-west-2.amazonaws.com/master/{COMMIT_HASH}/ray-{RAY_VERSION}-{PYTHON_VERSION}-{PYTHON_VERSION}m-{OS_VERSION}_intel.whl
For example, here are the Ray 1.1.0.dev0 wheels for Python 3.5, MacOS for commit ``a0ba4499ac645c9d3e82e68f3a281e48ad57f873``:
For example, here are the Ray 1.2.0.dev0 wheels for Python 3.5, MacOS for commit ``a0ba4499ac645c9d3e82e68f3a281e48ad57f873``:
.. code-block:: bash
pip install https://ray-wheels.s3-us-west-2.amazonaws.com/master/a0ba4499ac645c9d3e82e68f3a281e48ad57f873/ray-1.1.0.dev0-cp35-cp35m-macosx_10_13_intel.whl
pip install https://ray-wheels.s3-us-west-2.amazonaws.com/master/a0ba4499ac645c9d3e82e68f3a281e48ad57f873/ray-1.2.0.dev0-cp35-cp35m-macosx_10_13_intel.whl
.. _ray-install-java:
@@ -139,7 +140,7 @@ The latest Ray Java snapshot can be found in `sonatype repository <https://oss.s
.. note::
When you run ``pip install`` to install Ray, Java jars are installed as well. The above dependencies are only used to build your Java code and to run your code in local or single machine mode.
When you run ``pip install`` to install Ray, Java jars are installed as well. The above dependencies are only used to build your Java code and to run your code in local mode.
If you want to run your Java code in a multi-node Ray cluster, it's better to exclude Ray jars when packaging your code to avoid jar conficts if the versions (installed Ray with ``pip install`` and maven dependencies) don't match.
@@ -237,13 +238,45 @@ However, should you need to build from source, follow :ref:`these instructions f
Docker Source Images
--------------------
Most users should pull a Docker image from the Ray Docker Hub.
Most users should pull a Docker image from the `Ray Docker Hub. <https://hub.docker.com/r/rayproject/>`_
- The ``rayproject/ray`` image has ray and all required dependencies. It comes with anaconda and Python 3.7.
- The ``rayproject/autoscaler`` image has the above features as well as many additional libraries.
- The ``rayproject/ray`` `image has ray and all required dependencies. It comes with anaconda and Python 3.7. <https://hub.docker.com/r/rayproject/ray>`_
- The ``rayproject/ray-ml`` `image has the above features as well as many additional libraries. <https://hub.docker.com/r/rayproject/ray-ml>`_
- The ``rayproject/base-deps`` and ``rayproject/ray-deps`` are for the linux and python dependencies respectively.
These images are tagged by their release number (or commit hash for nightlies) as well as a ``"-gpu"`` if they are GPU compatible.
Image releases are `tagged` using the following format:
.. list-table::
:widths: 25 50
:header-rows: 1
* - Tag
- Description
* - latest
- The most recent Ray release.
* - 1.x.x
- A specific Ray release.
* - nightly
- The most recent Ray build (the most recent commit on Github ``master``)
* - Git SHA
- A specific nightly build (uses a SHA from the Github ``master``).
Each tag has `variants` that add or change functionality:
.. list-table::
:widths: 16 40
:header-rows: 1
* - Variant
- Description
* - -gpu
- These are based off of an NVIDIA CUDA image. They require the Nvidia Docker Runtime.
* - -cpu
- These are based off of an Ubuntu image.
* - <no tag>
- Aliases to ``-cpu`` tagged images
If you want to tweak some aspect of these images and build them locally, refer to the following script:
@@ -308,3 +341,32 @@ that you've cloned the git repository.
.. code-block:: bash
python -m pytest -v python/ray/tests/test_mini.py
Troubleshooting
---------------
If importing Ray (``python3 -c "import ray"``) in your development clone results
in this error:
.. code-block:: python
Traceback (most recent call last):
File "<string>", line 1, in <module>
File ".../ray/python/ray/__init__.py", line 63, in <module>
import ray._raylet # noqa: E402
File "python/ray/_raylet.pyx", line 98, in init ray._raylet
import ray.memory_monitor as memory_monitor
File ".../ray/python/ray/memory_monitor.py", line 9, in <module>
import psutil # noqa E402
File ".../ray/python/ray/thirdparty_files/psutil/__init__.py", line 159, in <module>
from . import _psosx as _psplatform
File ".../ray/python/ray/thirdparty_files/psutil/_psosx.py", line 15, in <module>
from . import _psutil_osx as cext
ImportError: cannot import name '_psutil_osx' from partially initialized module 'psutil' (most likely due to a circular import) (.../ray/python/ray/thirdparty_files/psutil/__init__.py)
Then you should run the following commands:
.. code-block:: bash
rm -rf python/ray/thirdparty_files/
python3 -m pip install setproctitle
doc/source/placement-group.rst
-1
View File
@@ -124,7 +124,6 @@ Let's create a placement group. Recall that each bundle is a collection of resou
- "CPU" will correspond with `num_cpus` as used in `ray.remote`
- "GPU" will correspond with `num_gpus` as used in `ray.remote`
- "MEM" will correspond with `memory` as used in `ray.remote`
- Other resources will correspond with `resources` as used in `ray.remote`.
Once the placement group reserves resources, original resources are unavailable until the placement group is removed. For example:
doc/source/ray-overview/index.rst
+1 -1
View File
@@ -196,7 +196,7 @@ RLlib Quick Start
.. code-block:: bash
pip install tensorflow # or tensorflow-gpu
pip install ray[rllib] # also recommended: ray[debug]
pip install ray[rllib]
.. code-block:: python
doc/source/ray-overview/involvement.rst
+1 -1
View File
@@ -1,7 +1,7 @@
Ray is more than a framework for distributed applications but also an active community of developers,
researchers, and folks that love machine learning. Here's a list of tips for getting involved with the Ray community:
- Join our `community slack <https://forms.gle/9TSdDYUgxYs8SA9e8>`_ to discuss Ray!
- Join our `community Slack <https://forms.gle/9TSdDYUgxYs8SA9e8>`_ to discuss Ray!
- Star and follow us on `on GitHub`_.
- To post questions or feature requests, check out the `Discussion Board`_!
- Follow us and spread the word on `Twitter`_!
doc/source/rllib.rst
+1 -1
View File
@@ -23,7 +23,7 @@ RLlib has extra dependencies on top of ``ray``. First, you'll need to install ei
.. code-block:: bash
pip install 'ray[rllib]' # also recommended: ray[debug]
pip install 'ray[rllib]'
Then, you can try out training in the following equivalent ways:
doc/source/serve/advanced.rst
+17 -5
View File
@@ -81,6 +81,12 @@ If you *do* want to enable this parallelism in your Serve backend, just set OMP_
client.create_backend("parallel_backend", MyBackend, 12)
.. note::
Some other libraries may not respect ``OMP_NUM_THREADS`` and have their own way to configure parallelism.
For example, if you're using OpenCV, you'll need to manually set the number of threads using ``cv2.setNumThreads(num_threads)`` (set to 0 to disable multi-threading).
You can check the configuration using ``cv2.getNumThreads()`` and ``cv2.getNumberOfCPUs()``.
.. _serve-batching:
Batching to improve performance
@@ -306,12 +312,18 @@ and another named ``ray-tf2`` with Ray Serve and Tensorflow 2. The Ray and
python versions must be the same in both environments. To specify
an environment for a backend to use, simply pass the environment name in to
:mod:`client.create_backend <ray.serve.api.Client.create_backend>`
as shown below. Be sure to run the script in an activated conda environment
(not required to be ``ray-tf1`` or ``ray-tf2``).
as shown below.
.. literalinclude:: ../../../python/ray/serve/examples/doc/conda_env.py
Alternatively, you may omit the argument ``env`` and call
:mod:`client.create_backend <ray.serve.api.Client.create_backend>`
from a script running in the conda environment you want the backend to run in.
.. warning::
The script must be run in an activated conda environment (not required to be
``ray-tf1`` or ``ray-tf2``). We hope to remove this restriction in the
future.
.. note::
If the argument ``env`` is omitted, backends will be started in the same
conda environment as the caller of
:mod:`client.create_backend <ray.serve.api.Client.create_backend>` by
default.
doc/source/serve/index.rst
+1 -1
View File
@@ -37,7 +37,7 @@ Since Serve is built on Ray, it also allows you to scale to many machines, in yo
Installation
============
Ray Serve supports Python versions 3.6 and higher. To install Ray Serve:
Ray Serve supports Python versions 3.6 through 3.8. To install Ray Serve:
.. code-block:: bash
doc/source/serve/key-concepts.rst
+5 -1
View File
@@ -19,7 +19,11 @@ Backends
Backends define the implementation of your business logic or models that will handle requests when queries come in to :ref:`serve-endpoint`.
In order to support seamless scalability backends can have many replicas, which are individual processes running in the Ray cluster to handle requests.
To define a backend, first you must define the "handler" or the business logic you'd like to respond with.
The handler should take as input a `Flask Request object <https://flask.palletsprojects.com/en/1.1.x/api/?highlight=request#flask.Request>`_ and return any JSON-serializable object as output.
The handler should take as input a `Flask Request object <https://flask.palletsprojects.com/en/1.1.x/api/?highlight=request#flask.Request>`_.
The handler should return any JSON-serializable object as output. For a more customizable response type, the handler may return a
`Starlette Response object <https://www.starlette.io/responses/>`_.
In the future, Ray Serve will support `Starlette Request objects <https://www.starlette.io/requests/>`_ as input as well.
A backend is defined using :mod:`client.create_backend <ray.serve.api.Client.create_backend>`, and the implementation can be defined as either a function or a class.
Use a function when your response is stateless and a class when you might need to maintain some state (like a model).
When using a class, you can specify arguments to be passed to the constructor in :mod:`client.create_backend <ray.serve.api.Client.create_backend>`, shown below.
doc/source/starting-ray.rst
+19 -10
View File
@@ -125,25 +125,34 @@ Use ``ray start`` from the CLI to start a 1 node ray runtime on a machine. This
...
You can connect to this Ray runtime by starting a driver process on the same node as where you ran ``ray start``:
.. tabs::
.. group-tab:: python
.. code-tab:: python
You can connect to this Ray runtime by starting a Python process that calls the following on the same node as where you ran ``ray start``:
.. code-block:: python
# This must
import ray
ray.init(address='auto')
# This must
import ray
ray.init(address='auto')
.. group-tab:: java
.. code-block:: java
If you want to run Java code, you need to specify the classpath via the ``--code-search-path`` option. See :ref:`code_search_path` for more details.
import io.ray.api.Ray;
public class MyRayApp {
public static void main(String[] args) {
Ray.init();
...
}
}
.. code-block:: bash
$ ray start ... --code-search-path=/path/to/jars
java -classpath <classpath> \
-Dray.address=<address> \
<classname> <args>
You can connect other nodes to the head node, creating a Ray cluster by also calling ``ray start`` on those nodes. See :ref:`manual-cluster` for more details. Calling ``ray.init(address="auto")`` on any of the cluster machines will connect to the ray cluster.
doc/source/tune/api_docs/execution.rst
-8
View File
@@ -23,14 +23,6 @@ tune.with_parameters
.. autofunction:: ray.tune.with_parameters
.. _tune-stop-ref:
Stopper (tune.Stopper)
----------------------
.. autoclass:: ray.tune.Stopper
:members: __call__, stop_all
.. _tune-sync-config:
tune.SyncConfig
doc/source/tune/api_docs/overview.rst
+1
View File
@@ -21,6 +21,7 @@ on `Github`_.
suggestion.rst
schedulers.rst
sklearn.rst
stoppers.rst
logging.rst
integration.rst
internals.rst
doc/source/tune/api_docs/stoppers.rst
+46
View File
@@ -0,0 +1,46 @@
.. _tune-stoppers:
Stopping mechanisms (tune.stopper)
==================================
In addition to Trial Schedulers like :ref:`ASHA <tune-scheduler-hyperband>`, where a number of
trials are stopped if they perform subpar, Ray Tune also supports custom stopping mechanisms to stop trials early. For instance, stopping mechanisms can specify to stop trials when they reached a plateau and the metric
doesn't change anymore.
Ray Tune comes with several stopping mechanisms out of the box. For custom stopping behavior, you can
inherit from the :class:`Stopper <ray.tune.Stopper>` class.
Other stopping behaviors are described :ref:`in the user guide <tune-stopping>`.
.. contents::
:local:
:depth: 1
.. _tune-stop-ref:
Stopper (tune.Stopper)
----------------------
.. autoclass:: ray.tune.Stopper
:members: __call__, stop_all
MaximumIterationStopper (tune.stopper.MaximumIterationStopper)
--------------------------------------------------------------
.. autoclass:: ray.tune.stopper.MaximumIterationStopper
ExperimentPlateauStopper (tune.stopper.ExperimentPlateauStopper)
----------------------------------------------------------------
.. autoclass:: ray.tune.stopper.ExperimentPlateauStopper
TrialPlateauStopper (tune.stopper.TrialPlateauStopper)
------------------------------------------------------
.. autoclass:: ray.tune.stopper.TrialPlateauStopper
TimeoutStopper (tune.stopper.TimeoutStopper)
--------------------------------------------
.. autoclass:: ray.tune.stopper.TimeoutStopper
doc/source/tune/user-guide.rst
+7 -3
View File
@@ -305,7 +305,9 @@ and passed to your trainable as a parameter.
Stopping Trials
---------------
You can control when trials are stopped early by passing the ``stop`` argument to ``tune.run``. This argument takes either a dictionary or a function.
You can control when trials are stopped early by passing the ``stop`` argument to ``tune.run``.
This argument takes, a dictionary, a function, or a :class:`Stopper <ray.tune.stopper.Stopper>` class
as an argument.
If a dictionary is passed in, the keys may be any field in the return result of ``tune.report`` in the Function API or ``step()`` (including the results from ``step`` and auto-filled metrics).
@@ -329,7 +331,7 @@ For more flexibility, you can pass in a function instead. If a function is passe
tune.run(my_trainable, stop=stopper)
Finally, you can implement the ``Stopper`` abstract class for stopping entire experiments. For example, the following example stops all trials after the criteria is fulfilled by any individual trial, and prevents new ones from starting:
Finally, you can implement the :class:`Stopper <ray.tune.stopper.Stopper>` abstract class for stopping entire experiments. For example, the following example stops all trials after the criteria is fulfilled by any individual trial, and prevents new ones from starting:
.. code-block:: python
@@ -352,7 +354,9 @@ Finally, you can implement the ``Stopper`` abstract class for stopping entire ex
tune.run(my_trainable, stop=stopper)
Note that in the above example the currently running trials will not stop immediately but will do so once their current iterations are complete. See the :ref:`tune-stop-ref` documentation.
Note that in the above example the currently running trials will not stop immediately but will do so once their current iterations are complete.
Ray Tune comes with a set of out-of-the-box stopper classes. See the :ref:`Stopper <tune-stoppers>` documentation.
.. _tune-logging:
doc/source/xgboost-ray.rst
-18
View File
@@ -143,21 +143,3 @@ the `examples folder <https://github.com/ray-project/xgboost_ray/tree/master/exa
* `[download dataset (2.6 GB)] <https://archive.ics.uci.edu/ml/machine-learning-databases/00280/HIGGS.csv.gz>`__
* `HIGGS classification example with Parquet <https://github.com/ray-project/xgboost_ray/tree/master/examples/higgs_parquet.py>`__ (uses the same dataset)
* `Test data classification <https://github.com/ray-project/xgboost_ray/tree/master/examples/train_on_test_data.py>`__ (uses a self-generated dataset)
Package Reference
-----------------
Training/Validation
~~~~~~~~~~~~~~~~~~~
.. autoclass:: ray.util.xgboost.RayParams
.. autofunction:: ray.util.xgboost.train
.. autofunction:: ray.util.xgboost.predict
RayDMatrix
~~~~~~~~~~
.. autoclass:: ray.util.xgboost.RayDMatrix
docker/ray-ml/Dockerfile
+2 -3
View File
@@ -13,11 +13,10 @@ RUN sudo apt-get update \
libgtk2.0-dev \
zlib1g-dev \
libgl1-mesa-dev \
&& $HOME/anaconda3/bin/pip --no-cache-dir install -r requirements.txt \
&& $HOME/anaconda3/bin/pip --no-cache-dir install -r requirements_ml_docker.txt \
&& $HOME/anaconda3/bin/pip --use-deprecated=legacy-resolver --no-cache-dir install -r requirements.txt \
&& $HOME/anaconda3/bin/pip --use-deprecated=legacy-resolver --no-cache-dir install -r requirements_ml_docker.txt \
# Remove dataclasses & typing because they are included in Py3.7
&& $HOME/anaconda3/bin/pip uninstall dataclasses typing -y \
&& sudo rm requirements.txt && sudo rm requirements_ml_docker.txt \
&& sudo apt-get remove cmake gcc -y \
&& sudo apt-get clean
java/BUILD.bazel
+1 -22
View File
@@ -70,6 +70,7 @@ define_java_module(
visibility = ["//visibility:public"],
deps = [
":io_ray_ray_api",
"@maven//:com_google_code_gson_gson",
"@maven//:com_google_guava_guava",
"@maven//:com_google_protobuf_protobuf_java",
"@maven//:com_typesafe_config",
@@ -134,27 +135,12 @@ filegroup(
],
)
native_java_binary("runtime", "raylet", "//:raylet")
native_java_binary("runtime", "plasma_store_server", "//:plasma_store_server")
native_java_binary("runtime", "redis-server", "//:redis-server")
native_java_binary("runtime", "gcs_server", "//:gcs_server")
native_java_binary("runtime", "libray_redis_module.so", "//:libray_redis_module.so")
native_java_library("runtime", "core_worker_library_java", "//:libcore_worker_library_java.so")
filegroup(
name = "java_native_deps",
srcs = [
":core_worker_library_java",
":gcs_server",
":libray_redis_module.so",
":plasma_store_server",
":raylet",
":redis-server",
],
)
@@ -252,13 +238,6 @@ genrule(
WORK_DIR="$$(pwd)"
rm -rf "$$WORK_DIR/python/ray/jars" && mkdir -p "$$WORK_DIR/python/ray/jars"
cp -f $(location //java:ray_dist_deploy.jar) "$$WORK_DIR/python/ray/jars/ray_dist.jar"
chmod +w "$$WORK_DIR/python/ray/jars/ray_dist.jar"
zip -d "$$WORK_DIR/python/ray/jars/ray_dist.jar" \
"native/*/gcs_server" \
"native/*/libray_redis_module.so" \
"native/*/plasma_store_server" \
"native/*/raylet" \
"native/*/redis-server"
date > $@
""",
local = 1,
java/api/src/main/java/io/ray/api/runtime/RayRuntime.java
+8
View File
@@ -221,4 +221,12 @@ public interface RayRuntime {
* @param id Id of the placement group.
*/
void removePlacementGroup(PlacementGroupId id);
/**
* Wait for the placement group to be ready within the specified time.
* @param id Id of placement group.
* @param timeoutMs Timeout in milliseconds.
* @return True if the placement group is created. False otherwise.
*/
boolean waitPlacementGroupReady(PlacementGroupId id, int timeoutMs);
}
java/api/src/main/java/io/ray/api/runtimecontext/RuntimeContext.java
-10
View File
@@ -28,16 +28,6 @@ public interface RuntimeContext {
*/
boolean wasCurrentActorRestarted();
/**
* Get the raylet socket name.
*/
String getRayletSocketName();
/**
* Get the object store socket name.
*/
String getObjectStoreSocketName();
/**
* Return true if Ray is running in single-process mode, false if Ray is running in cluster mode.
*/
java/runtime/pom.xml
+5
View File
@@ -39,6 +39,11 @@
<artifactId>ray-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.5</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
java/runtime/src/main/java/io/ray/runtime/AbstractRayRuntime.java
+5
View File
@@ -200,6 +200,11 @@ public abstract class AbstractRayRuntime implements RayRuntimeInternal {
return gcsClient.getAllPlacementGroupInfo();
}
@Override
public boolean waitPlacementGroupReady(PlacementGroupId id, int timeoutMs) {
return taskSubmitter.waitPlacementGroupReady(id, timeoutMs);
}
@SuppressWarnings("unchecked")
@Override
public <T extends BaseActorHandle> T getActorHandle(ActorId actorId) {
java/runtime/src/main/java/io/ray/runtime/DefaultRayRuntimeFactory.java
+1 -1
View File
@@ -16,7 +16,7 @@ public class DefaultRayRuntimeFactory implements RayRuntimeFactory {
@Override
public RayRuntime createRayRuntime() {
RayConfig rayConfig = RayConfig.getInstance();
RayConfig rayConfig = RayConfig.create();
LoggingUtil.setupLogging(rayConfig);
Logger logger = LoggerFactory.getLogger(DefaultRayRuntimeFactory.class);
java/runtime/src/main/java/io/ray/runtime/RayDevRuntime.java
-1
View File
@@ -57,7 +57,6 @@ public class RayDevRuntime extends AbstractRayRuntime {
taskSubmitter = null;
}
taskExecutor = null;
RayConfig.reset();
}
@Override
java/runtime/src/main/java/io/ray/runtime/RayNativeRuntime.java
+86 -120
View File
@@ -5,10 +5,8 @@ import io.ray.api.BaseActorHandle;
import io.ray.api.id.ActorId;
import io.ray.api.id.JobId;
import io.ray.api.id.UniqueId;
import io.ray.api.runtimecontext.NodeInfo;
import io.ray.runtime.config.RayConfig;
import io.ray.runtime.context.NativeWorkerContext;
import io.ray.runtime.exception.RayException;
import io.ray.runtime.exception.RayIntentionalSystemExitException;
import io.ray.runtime.gcs.GcsClient;
import io.ray.runtime.gcs.GcsClientOptions;
@@ -22,15 +20,12 @@ import io.ray.runtime.task.NativeTaskSubmitter;
import io.ray.runtime.task.TaskExecutor;
import io.ray.runtime.util.BinaryFileUtil;
import io.ray.runtime.util.JniUtils;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,7 +36,7 @@ public final class RayNativeRuntime extends AbstractRayRuntime {
private static final Logger LOGGER = LoggerFactory.getLogger(RayNativeRuntime.class);
private RunManager manager = null;
private boolean startRayHead = false;
/**
* In Java, GC runs in a standalone thread, and we can't control the exact
@@ -52,124 +47,101 @@ public final class RayNativeRuntime extends AbstractRayRuntime {
*/
private final ReadWriteLock shutdownLock = new ReentrantReadWriteLock();
public RayNativeRuntime(RayConfig rayConfig) {
super(rayConfig);
}
static {
LOGGER.debug("Loading native libraries.");
// Expose ray ABI symbols which may be depended by other shared
// libraries such as libstreaming_java.so.
// See BUILD.bazel:libcore_worker_library_java.so
final RayConfig rayConfig = RayConfig.getInstance();
if (rayConfig.getRedisAddress() != null && rayConfig.workerMode == WorkerType.DRIVER) {
// Fetch session dir from GCS if this is a driver that is connecting to the existing GCS.
private void updateSessionDir() {
if (rayConfig.workerMode == WorkerType.DRIVER) {
// Fetch session dir from GCS if this is a driver.
RedisClient client = new RedisClient(rayConfig.getRedisAddress(), rayConfig.redisPassword);
final String sessionDir = client.get("session_dir", null);
Preconditions.checkNotNull(sessionDir);
rayConfig.setSessionDir(sessionDir);
}
JniUtils.loadLibrary(BinaryFileUtil.CORE_WORKER_JAVA_LIBRARY, true);
LOGGER.debug("Native libraries loaded.");
try {
FileUtils.forceMkdir(new File(rayConfig.logDir));
} catch (IOException e) {
throw new RuntimeException("Failed to create the log directory.", e);
}
}
public RayNativeRuntime(RayConfig rayConfig) {
super(rayConfig);
loadConfigFromGcs(rayConfig);
}
private static void loadConfigFromGcs(RayConfig rayConfig) {
if (rayConfig.getRedisAddress() != null) {
GcsClient tempGcsClient =
new GcsClient(rayConfig.getRedisAddress(), rayConfig.redisPassword);
for (Map.Entry<String, String> entry :
tempGcsClient.getInternalConfig().entrySet()) {
rayConfig.rayletConfigParameters.put(entry.getKey(), entry.getValue());
}
if (rayConfig.workerMode == WorkerType.DRIVER) {
// Keep this method logic in sync with `services.get_address_info_from_redis_helper`
int numRetries = 5;
int retryCount = 0;
boolean configLoaded = false;
while (retryCount++ < numRetries) {
for (NodeInfo nodeInfo : tempGcsClient.getAllNodeInfo()) {
if (rayConfig.nodeIp.equals(nodeInfo.nodeAddress) ||
(nodeInfo.nodeAddress.equals("127.0.0.1") &&
rayConfig.nodeIp.equals(rayConfig.getRedisAddress()))) {
rayConfig.objectStoreSocketName = nodeInfo.objectStoreSocketName;
rayConfig.rayletSocketName = nodeInfo.rayletSocketName;
rayConfig.nodeManagerPort = nodeInfo.nodeManagerPort;
configLoaded = true;
break;
}
}
if (!configLoaded) {
LOGGER.warn("Some processes that the driver needs to connect to have " +
"not registered with Redis, so retrying. Have you run " +
"'ray start' on this node?");
try {
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
} else {
break;
}
}
if (!configLoaded) {
throw new RayException("Some processes that the driver needs to connect to have " +
"not registered with Redis. Have you run 'ray start' on this node?");
}
}
private void loadConfigFromGcs() {
rayConfig.rayletConfigParameters.clear();
for (Map.Entry<String, String> entry : gcsClient.getInternalConfig().entrySet()) {
rayConfig.rayletConfigParameters.put(entry.getKey(), entry.getValue());
}
}
@Override
public void start() {
if (rayConfig.getRedisAddress() == null) {
manager = new RunManager(rayConfig);
manager.startRayProcesses(true);
try {
if (rayConfig.workerMode == WorkerType.DRIVER && rayConfig.getRedisAddress() == null) {
// Set it to true before `RunManager.startRayHead` so `Ray.shutdown()` can still kill
// Ray processes even if `Ray.init()` failed.
startRayHead = true;
RunManager.startRayHead(rayConfig);
}
Preconditions.checkNotNull(rayConfig.getRedisAddress());
updateSessionDir();
// Expose ray ABI symbols which may be depended by other shared
// libraries such as libstreaming_java.so.
// See BUILD.bazel:libcore_worker_library_java.so
Preconditions.checkNotNull(rayConfig.sessionDir);
JniUtils.loadLibrary(rayConfig.sessionDir, BinaryFileUtil.CORE_WORKER_JAVA_LIBRARY, true);
if (rayConfig.workerMode == WorkerType.DRIVER) {
RunManager.getAddressInfoAndFillConfig(rayConfig);
}
gcsClient = new GcsClient(rayConfig.getRedisAddress(), rayConfig.redisPassword);
loadConfigFromGcs();
if (rayConfig.getJobId() == JobId.NIL) {
rayConfig.setJobId(gcsClient.nextJobId());
}
int numWorkersPerProcess =
rayConfig.workerMode == WorkerType.DRIVER ? 1 : rayConfig.numWorkersPerProcess;
byte[] serializedJobConfig = null;
if (rayConfig.workerMode == WorkerType.DRIVER) {
JobConfig.Builder jobConfigBuilder =
JobConfig.newBuilder()
.setNumJavaWorkersPerProcess(rayConfig.numWorkersPerProcess)
.addAllJvmOptions(rayConfig.jvmOptionsForJavaWorker)
.putAllWorkerEnv(rayConfig.workerEnv)
.addAllCodeSearchPath(rayConfig.codeSearchPath);
serializedJobConfig = jobConfigBuilder.build().toByteArray();
}
Map<String, String> rayletConfigStringMap = new HashMap<>();
for (Map.Entry<String, Object> entry : rayConfig.rayletConfigParameters.entrySet()) {
rayletConfigStringMap.put(entry.getKey(), entry.getValue().toString());
}
nativeInitialize(rayConfig.workerMode.getNumber(),
rayConfig.nodeIp, rayConfig.getNodeManagerPort(),
rayConfig.workerMode == WorkerType.DRIVER ? System.getProperty("user.dir") : "",
rayConfig.objectStoreSocketName, rayConfig.rayletSocketName,
(rayConfig.workerMode == WorkerType.DRIVER ? rayConfig.getJobId() : JobId.NIL).getBytes(),
new GcsClientOptions(rayConfig), numWorkersPerProcess,
rayConfig.logDir, rayletConfigStringMap, serializedJobConfig);
taskExecutor = new NativeTaskExecutor(this);
workerContext = new NativeWorkerContext();
objectStore = new NativeObjectStore(workerContext, shutdownLock);
taskSubmitter = new NativeTaskSubmitter();
LOGGER.debug("RayNativeRuntime started with store {}, raylet {}",
rayConfig.objectStoreSocketName, rayConfig.rayletSocketName);
} catch (Exception e) {
if (startRayHead) {
try {
RunManager.stopRay();
} catch (Exception e2) {
// Ignore
}
}
throw e;
}
gcsClient = new GcsClient(rayConfig.getRedisAddress(), rayConfig.redisPassword);
if (rayConfig.getJobId() == JobId.NIL) {
rayConfig.setJobId(gcsClient.nextJobId());
}
int numWorkersPerProcess =
rayConfig.workerMode == WorkerType.DRIVER ? 1 : rayConfig.numWorkersPerProcess;
byte[] serializedJobConfig = null;
if (rayConfig.workerMode == WorkerType.DRIVER) {
JobConfig.Builder jobConfigBuilder =
JobConfig.newBuilder()
.setNumJavaWorkersPerProcess(rayConfig.numWorkersPerProcess)
.addAllJvmOptions(rayConfig.jvmOptionsForJavaWorker)
.putAllWorkerEnv(rayConfig.workerEnv)
.addAllCodeSearchPath(rayConfig.codeSearchPath);
serializedJobConfig = jobConfigBuilder.build().toByteArray();
}
// TODO(qwang): Get object_store_socket_name and raylet_socket_name from Redis.
nativeInitialize(rayConfig.workerMode.getNumber(),
rayConfig.nodeIp, rayConfig.getNodeManagerPort(),
rayConfig.workerMode == WorkerType.DRIVER ? System.getProperty("user.dir") : "",
rayConfig.objectStoreSocketName, rayConfig.rayletSocketName,
(rayConfig.workerMode == WorkerType.DRIVER ? rayConfig.getJobId() : JobId.NIL).getBytes(),
new GcsClientOptions(rayConfig), numWorkersPerProcess,
rayConfig.logDir, rayConfig.rayletConfigParameters, serializedJobConfig);
taskExecutor = new NativeTaskExecutor(this);
workerContext = new NativeWorkerContext();
objectStore = new NativeObjectStore(workerContext, shutdownLock);
taskSubmitter = new NativeTaskSubmitter();
LOGGER.debug("RayNativeRuntime started with store {}, raylet {}",
rayConfig.objectStoreSocketName, rayConfig.rayletSocketName);
}
@Override
@@ -183,27 +155,21 @@ public final class RayNativeRuntime extends AbstractRayRuntime {
try {
if (rayConfig.workerMode == WorkerType.DRIVER) {
nativeShutdown();
if (null != manager) {
manager.cleanup();
manager = null;
if (startRayHead) {
startRayHead = false;
RunManager.stopRay();
}
}
if (null != gcsClient) {
gcsClient.destroy();
gcsClient = null;
}
RayConfig.reset();
LOGGER.debug("RayNativeRuntime shutdown");
} finally {
writeLock.unlock();
}
}
// For test purpose only
public RunManager getRunManager() {
return manager;
}
@Override
public void setResource(String resourceName, double capacity, UniqueId nodeId) {
Preconditions.checkArgument(Double.compare(capacity, 0) >= 0);
java/runtime/src/main/java/io/ray/runtime/config/RayConfig.java
+42 -166
View File
@@ -2,7 +2,6 @@ package io.ray.runtime.config;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigException;
@@ -12,17 +11,14 @@ import com.typesafe.config.ConfigValue;
import io.ray.api.id.JobId;
import io.ray.runtime.generated.Common.WorkerType;
import io.ray.runtime.util.NetworkUtil;
import io.ray.runtime.util.ResourceUtil;
import java.io.File;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
/**
* Configurations of Ray runtime.
@@ -33,13 +29,6 @@ public class RayConfig {
public static final String DEFAULT_CONFIG_FILE = "ray.default.conf";
public static final String CUSTOM_CONFIG_FILE = "ray.conf";
private static final Random RANDOM = new Random();
private static final DateTimeFormatter DATE_TIME_FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss");
private static final String DEFAULT_TEMP_DIR = "/tmp/ray";
private Config config;
/**
@@ -48,54 +37,25 @@ public class RayConfig {
public final String nodeIp;
public final WorkerType workerMode;
public final RunMode runMode;
public final Map<String, Double> resources;
private JobId jobId;
public String sessionDir;
public String logDir;
public final List<String> libraryPath;
public final List<String> classpath;
public final List<String> jvmParameters;
private String redisAddress;
private String redisIp;
private Integer redisPort;
public final int headRedisPort;
public final int[] redisShardPorts;
public final int numberRedisShards;
public final String headRedisPassword;
public final String redisPassword;
// RPC socket name of object store.
public String objectStoreSocketName;
public final Long objectStoreSize;
// RPC socket name of Raylet.
public String rayletSocketName;
// Listening port for node manager.
public int nodeManagerPort;
public final Map<String, String> rayletConfigParameters;
public final Map<String, Object> rayletConfigParameters;
public List<String> codeSearchPath;
public final String pythonWorkerCommand;
public final List<String> codeSearchPath;
private static volatile RayConfig instance = null;
public static RayConfig getInstance() {
if (instance == null) {
synchronized (RayConfig.class) {
if (instance == null) {
instance = RayConfig.create();
}
}
}
return instance;
}
public static void reset() {
synchronized (RayConfig.class) {
instance = null;
}
}
public final List<String> headArgs;
public final int numWorkersPerProcess;
@@ -140,15 +100,6 @@ public class RayConfig {
} else {
nodeIp = NetworkUtil.getIpAddress(null);
}
// Resources.
resources = ResourceUtil.getResourcesMapFromString(
config.getString("ray.resources"));
if (isDriver) {
if (!resources.containsKey("CPU")) {
int numCpu = Runtime.getRuntime().availableProcessors();
resources.put("CPU", numCpu * 1.0);
}
}
// Job id.
String jobId = config.getString("ray.job.id");
if (!jobId.isEmpty()) {
@@ -168,25 +119,16 @@ public class RayConfig {
}
}
workerEnv = workerEnvBuilder.build();
updateSessionDir();
// Object store configurations.
objectStoreSize = config.getBytes("ray.object-store.size");
updateSessionDir(null);
// Library path.
libraryPath = config.getStringList("ray.library.path");
// Custom classpath.
classpath = config.getStringList("ray.classpath");
// Custom worker jvm parameters.
if (config.hasPath("ray.worker.jvm-parameters")) {
jvmParameters = config.getStringList("ray.worker.jvm-parameters");
} else {
jvmParameters = ImmutableList.of();
// Object store socket name.
if (config.hasPath("ray.object-store.socket-name")) {
objectStoreSocketName = config.getString("ray.object-store.socket-name");
}
if (config.hasPath("ray.worker.python-command")) {
pythonWorkerCommand = config.getString("ray.worker.python-command");
} else {
pythonWorkerCommand = null;
// Raylet socket name.
if (config.hasPath("ray.raylet.socket-name")) {
rayletSocketName = config.getString("ray.raylet.socket-name");
}
// Redis configurations.
@@ -198,17 +140,6 @@ public class RayConfig {
this.redisAddress = null;
}
if (config.hasPath("ray.redis.head-port")) {
headRedisPort = config.getInt("ray.redis.head-port");
} else {
headRedisPort = NetworkUtil.getUnusedPort();
}
numberRedisShards = config.getInt("ray.redis.shard-number");
redisShardPorts = new int[numberRedisShards];
for (int i = 0; i < numberRedisShards; i++) {
redisShardPorts[i] = NetworkUtil.getUnusedPort();
}
headRedisPassword = config.getString("ray.redis.head-password");
redisPassword = config.getString("ray.redis.password");
// Raylet node manager port.
if (config.hasPath("ray.raylet.node-manager-port")) {
@@ -216,7 +147,6 @@ public class RayConfig {
} else {
Preconditions.checkState(workerMode != WorkerType.WORKER,
"Worker started by raylet should accept the node manager port from raylet.");
nodeManagerPort = NetworkUtil.getUnusedPort();
}
// Raylet parameters.
@@ -224,39 +154,33 @@ public class RayConfig {
Config rayletConfig = config.getConfig("ray.raylet.config");
for (Map.Entry<String, ConfigValue> entry : rayletConfig.entrySet()) {
Object value = entry.getValue().unwrapped();
rayletConfigParameters.put(entry.getKey(), value == null ? "" : value.toString());
if (value != null) {
if (value instanceof String) {
String valueString = (String) value;
Boolean booleanValue = BooleanUtils.toBooleanObject(valueString);
if (booleanValue != null) {
value = booleanValue;
} else if (NumberUtils.isParsable(valueString)) {
value = NumberUtils.createNumber(valueString);
}
}
rayletConfigParameters.put(entry.getKey(), value);
}
}
// Job code search path.
String codeSearchPathString = null;
if (config.hasPath("ray.job.code-search-path")) {
codeSearchPath = Arrays.asList(
config.getString("ray.job.code-search-path").split(":"));
} else {
codeSearchPath = Collections.emptyList();
codeSearchPathString = config.getString("ray.job.code-search-path");
}
if (StringUtils.isEmpty(codeSearchPathString)) {
codeSearchPathString = System.getProperty("java.class.path");
}
codeSearchPath = Arrays.asList(codeSearchPathString.split(":"));
boolean enableMultiTenancy;
if (config.hasPath("ray.raylet.config.enable_multi_tenancy")) {
enableMultiTenancy =
Boolean.valueOf(config.getString("ray.raylet.config.enable_multi_tenancy"));
} else {
String envString = System.getenv("RAY_ENABLE_MULTI_TENANCY");
if (StringUtils.isNotBlank(envString)) {
enableMultiTenancy = "1".equals(envString);
} else {
enableMultiTenancy = true; // Default value
}
}
numWorkersPerProcess = config.getInt("ray.job.num-java-workers-per-process");
if (!enableMultiTenancy) {
if (!isDriver) {
numWorkersPerProcess = config.getInt("ray.raylet.config.num_workers_per_process_java");
} else {
numWorkersPerProcess = 1; // Actually this value isn't used in RayNativeRuntime.
}
} else {
numWorkersPerProcess = config.getInt("ray.job.num-java-workers-per-process");
}
headArgs = config.getStringList("ray.head-args");
// Validate config.
validate();
@@ -267,24 +191,12 @@ public class RayConfig {
Preconditions.checkState(this.redisAddress == null, "Redis address was already set");
this.redisAddress = redisAddress;
String[] ipAndPort = redisAddress.split(":");
Preconditions.checkArgument(ipAndPort.length == 2, "Invalid redis address.");
this.redisIp = ipAndPort[0];
this.redisPort = Integer.parseInt(ipAndPort[1]);
}
public String getRedisAddress() {
return redisAddress;
}
public String getRedisIp() {
return redisIp;
}
public Integer getRedisPort() {
return redisPort;
}
public void setJobId(JobId jobId) {
this.jobId = jobId;
}
@@ -298,11 +210,7 @@ public class RayConfig {
}
public void setSessionDir(String sessionDir) {
this.sessionDir = sessionDir;
}
public String getSessionDir() {
return sessionDir;
updateSessionDir(sessionDir);
}
public Config getInternalConfig() {
@@ -312,7 +220,8 @@ public class RayConfig {
/**
* Renders the config value as a HOCON string.
*/
public String render() {
@Override
public String toString() {
// These items might be dynamically generated or mutated at runtime.
// Explicitly include them.
Map<String, Object> dynamic = new HashMap<>();
@@ -321,24 +230,19 @@ public class RayConfig {
dynamic.put("ray.object-store.socket-name", objectStoreSocketName);
dynamic.put("ray.raylet.node-manager-port", nodeManagerPort);
dynamic.put("ray.address", redisAddress);
dynamic.put("ray.job.code-search-path", codeSearchPath);
Config toRender = ConfigFactory.parseMap(dynamic).withFallback(config);
return toRender.root().render(ConfigRenderOptions.concise());
}
private void updateSessionDir() {
private void updateSessionDir(String sessionDir) {
// session dir
if (workerMode == WorkerType.DRIVER) {
final int minBound = 100000;
final int maxBound = 999999;
final String sessionName = String.format("session_%s_%d", DATE_TIME_FORMATTER.format(
LocalDateTime.now()), RANDOM.nextInt(maxBound - minBound) + minBound);
sessionDir = String.format("%s/%s", DEFAULT_TEMP_DIR, sessionName);
} else if (workerMode == WorkerType.WORKER) {
sessionDir = removeTrailingSlash(config.getString("ray.session-dir"));
} else {
throw new RuntimeException("Unknown worker type.");
if (config.hasPath("ray.session-dir")) {
sessionDir = config.getString("ray.session-dir");
}
if (sessionDir != null) {
sessionDir = removeTrailingSlash(sessionDir);
}
this.sessionDir = sessionDir;
// Log dir.
String localLogDir = null;
@@ -350,34 +254,6 @@ public class RayConfig {
} else {
logDir = localLogDir;
}
// Object store socket name.
String localObjectStoreSocketName = null;
if (config.hasPath("ray.object-store.socket-name")) {
localObjectStoreSocketName = config.getString("ray.object-store.socket-name");
}
if (Strings.isNullOrEmpty(localObjectStoreSocketName)) {
objectStoreSocketName = String.format("%s/sockets/object_store", sessionDir);
} else {
objectStoreSocketName = localObjectStoreSocketName;
}
// Raylet socket name.
String localRayletSocketName = null;
if (config.hasPath("ray.raylet.socket-name")) {
localRayletSocketName = config.getString("ray.raylet.socket-name");
}
if (Strings.isNullOrEmpty(localRayletSocketName)) {
rayletSocketName = String.format("%s/sockets/raylet", sessionDir);
} else {
rayletSocketName = localRayletSocketName;
}
}
@Override
public String toString() {
return render();
}
/**
java/runtime/src/main/java/io/ray/runtime/context/RuntimeContextImpl.java
-10
View File
@@ -43,16 +43,6 @@ public class RuntimeContextImpl implements RuntimeContext {
return runtime.getGcsClient().wasCurrentActorRestarted(getCurrentActorId());
}
@Override
public String getRayletSocketName() {
return runtime.getRayConfig().rayletSocketName;
}
@Override
public String getObjectStoreSocketName() {
return runtime.getRayConfig().objectStoreSocketName;
}
@Override
public boolean isSingleProcess() {
return RunMode.SINGLE_PROCESS == runtime.getRayConfig().runMode;
java/runtime/src/main/java/io/ray/runtime/gcs/GcsClientOptions.java
+5 -2
View File
@@ -1,5 +1,6 @@
package io.ray.runtime.gcs;
import com.google.common.base.Preconditions;
import io.ray.runtime.config.RayConfig;
/**
@@ -11,8 +12,10 @@ public class GcsClientOptions {
public String password;
public GcsClientOptions(RayConfig rayConfig) {
ip = rayConfig.getRedisIp();
port = rayConfig.getRedisPort();
String[] ipAndPort = rayConfig.getRedisAddress().split(":");
Preconditions.checkArgument(ipAndPort.length == 2, "Invalid redis address.");
ip = ipAndPort[0];
port = Integer.parseInt(ipAndPort[1]);
password = rayConfig.redisPassword;
}
}
java/runtime/src/main/java/io/ray/runtime/gcs/GlobalStateAccessor.java
+1 -2
View File
@@ -82,8 +82,7 @@ public class GlobalStateAccessor {
public byte[] getPlacementGroupInfo(PlacementGroupId placementGroupId) {
synchronized (GlobalStateAccessor.class) {
Preconditions.checkNotNull(placementGroupId,
"PlacementGroupId can't be null when get placement group info.");
validateGlobalStateAccessorPointer();
return nativeGetPlacementGroupInfo(globalStateAccessorNativePointer,
placementGroupId.getBytes());
}
java/runtime/src/main/java/io/ray/runtime/placementgroup/PlacementGroupImpl.java
+10
View File
@@ -1,5 +1,6 @@
package io.ray.runtime.placementgroup;
import io.ray.api.Ray;
import io.ray.api.id.PlacementGroupId;
import io.ray.api.placementgroup.PlacementGroup;
import io.ray.api.placementgroup.PlacementGroupState;
@@ -49,6 +50,15 @@ public class PlacementGroupImpl implements PlacementGroup {
return state;
}
/**
* Wait for the placement group to be ready within the specified time.
* @param timeoutSeconds Timeout in seconds.
* @return True if the placement group is created. False otherwise.
*/
public boolean wait(int timeoutSeconds) {
return Ray.internal().waitPlacementGroupReady(id, timeoutSeconds);
}
/**
* A help class for create the placement group.
*/
java/runtime/src/main/java/io/ray/runtime/runner/RunManager.java
+75 -374
View File
@@ -1,31 +1,20 @@
package io.ray.runtime.runner;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import io.ray.runtime.config.RayConfig;
import io.ray.runtime.util.BinaryFileUtil;
import io.ray.runtime.util.ResourceUtil;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
/**
* Ray service management on one box.
@@ -34,97 +23,71 @@ public class RunManager {
private static final Logger LOGGER = LoggerFactory.getLogger(RunManager.class);
private static final String WORKER_CLASS = "io.ray.runtime.runner.worker.DefaultWorker";
private static final String SESSION_LATEST = "session_latest";
private RayConfig rayConfig;
private List<Pair<String, Process>> processes;
private static final int KILL_PROCESS_WAIT_TIMEOUT_SECONDS = 1;
public RunManager(RayConfig rayConfig) {
this.rayConfig = rayConfig;
processes = new ArrayList<>();
createTempDirs();
}
public void cleanup() {
// Terminate the processes in the reversed order of creating them.
// Because raylet needs to exit before object store, otherwise it
// cannot exit gracefully.
for (int i = processes.size() - 1; i >= 0; --i) {
Pair<String, Process> pair = processes.get(i);
terminateProcess(pair.getLeft(), pair.getRight());
}
}
public void terminateProcess(String name, Process p) {
int numAttempts = 0;
while (p.isAlive()) {
if (numAttempts == 0) {
LOGGER.debug("Terminating process {}.", name);
p.destroy();
} else {
LOGGER.debug("Terminating process {} forcibly.", name);
p.destroyForcibly();
}
try {
p.waitFor(KILL_PROCESS_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
} catch (InterruptedException e) {
LOGGER.warn("Got InterruptedException while waiting for process {}" +
" to be terminated.", name);
}
numAttempts++;
}
LOGGER.debug("Process {} is now terminated.", name);
}
private static final Pattern pattern = Pattern.compile("--address='([^']+)'");
/**
* Get processes by name. For test purposes only.
* Start the head node.
*/
public List<Process> getProcesses(String name) {
return processes.stream().filter(pair -> pair.getLeft().equals(name)).map(Pair::getRight)
.collect(Collectors.toList());
}
private void createTempDirs() {
public static void startRayHead(RayConfig rayConfig) {
LOGGER.debug("Starting ray runtime @ {}.", rayConfig.nodeIp);
List<String> command = new ArrayList<>();
command.add("ray");
command.add("start");
command.add("--head");
command.add("--redis-password");
command.add(rayConfig.redisPassword);
command.add("--system-config=" + new Gson().toJson(rayConfig.rayletConfigParameters));
command.addAll(rayConfig.headArgs);
String output;
try {
FileUtils.forceMkdir(new File(rayConfig.logDir));
FileUtils.forceMkdir(new File(rayConfig.rayletSocketName).getParentFile());
FileUtils.forceMkdir(new File(rayConfig.objectStoreSocketName).getParentFile());
// Remove session_latest first, and then create a new symbolic link for session_latest.
final String parentOfSessionDir = new File(rayConfig.sessionDir).getParent();
final File sessionLatest = new File(
String.format("%s/%s", parentOfSessionDir, SESSION_LATEST));
if (sessionLatest.exists()) {
sessionLatest.delete();
}
Files.createSymbolicLink(
Paths.get(sessionLatest.getAbsolutePath()),
Paths.get(rayConfig.sessionDir));
} catch (IOException e) {
LOGGER.error("Couldn't create temp directories.", e);
throw new RuntimeException(e);
output = runCommand(command);
} catch (Exception e) {
throw new RuntimeException("Failed to start Ray runtime.", e);
}
Matcher matcher = pattern.matcher(output);
if (matcher.find()) {
String redisAddress = matcher.group(1);
rayConfig.setRedisAddress(redisAddress);
} else {
throw new RuntimeException("Redis address is not found. output: " + output);
}
LOGGER.info("Ray runtime started @ {}.", rayConfig.nodeIp);
}
/**
* @return Log files for stdout and stderr.
* Stop ray.
*/
private Pair<File, File> getLogFiles(String logDir, String processName) {
int suffixIndex = 0;
while (true) {
String suffix = suffixIndex == 0 ? "" : "." + suffixIndex;
File stdout = new File(String.format("%s/%s%s.out", logDir, suffix, processName));
File stderr = new File(String.format("%s/%s%s.err", logDir, suffix, processName));
if (!stdout.exists() && !stderr.exists()) {
return ImmutablePair.of(stdout, stderr);
}
suffixIndex += 1;
public static void stopRay() {
List<String> command = new ArrayList<>();
command.add("ray");
command.add("stop");
command.add("--force");
try {
runCommand(command);
} catch (Exception e) {
throw new RuntimeException("Failed to stop ray.", e);
}
}
public static void getAddressInfoAndFillConfig(RayConfig rayConfig) {
// NOTE(kfstorm): This method depends on an internal Python API of ray to get the
// address info of the local node.
String script = String.format("import ray;"
+ " print(ray._private.services.get_address_info_from_redis("
+ "'%s', '%s', redis_password='%s', no_warning=True))",
rayConfig.getRedisAddress(), rayConfig.nodeIp, rayConfig.redisPassword);
List<String> command = Arrays.asList("python", "-c", script);
String output = null;
try {
output = runCommand(command);
JsonObject addressInfo = new JsonParser().parse(output).getAsJsonObject();
rayConfig.rayletSocketName = addressInfo.get("raylet_socket_name").getAsString();
rayConfig.objectStoreSocketName = addressInfo.get("object_store_address").getAsString();
rayConfig.nodeManagerPort = addressInfo.get("node_manager_port").getAsInt();
} catch (Exception e) {
throw new RuntimeException("Failed to get address info. Output: " + output, e);
}
}
@@ -132,284 +95,22 @@ public class RunManager {
* Start a process.
*
* @param command The command to start the process with.
* @param env Environment variables.
* @param name Process name.
*/
private void startProcess(List<String> command, Map<String, String> env, String name) {
private static String runCommand(List<String> command) throws IOException, InterruptedException {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Starting process {} with command: {}", name,
Joiner.on(" ").join(command));
LOGGER.debug("Starting process with command: {}", Joiner.on(" ").join(command));
}
ProcessBuilder builder = new ProcessBuilder(command);
String stdout = "";
String stderr = "";
// Set stdout and stderr paths.
Pair<File, File> logFiles = getLogFiles(rayConfig.logDir, name);
builder.redirectOutput(logFiles.getLeft());
builder.redirectError(logFiles.getRight());
// Set environment variables.
if (env != null && !env.isEmpty()) {
builder.environment().putAll(env);
}
Process p;
try {
p = builder.start();
} catch (IOException e) {
LOGGER.error("Failed to start process " + name, e);
throw new RuntimeException("Failed to start process " + name, e);
}
// Wait 1000 ms and check whether the process is alive.
try {
TimeUnit.MILLISECONDS.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
if (!p.isAlive()) {
String message = String.format("Failed to start %s. Exit code: %d.",
name, p.exitValue());
message += String.format(" Logs are redirected to %s and %s.", stdout, stderr);
throw new RuntimeException(message);
}
processes.add(Pair.of(name, p));
if (LOGGER.isDebugEnabled()) {
String message = String.format("%s process started.", name);
message += String.format(" Logs are redirected to %s and %s.", stdout, stderr);
LOGGER.debug(message);
ProcessBuilder builder = new ProcessBuilder(command).redirectErrorStream(true);
Process p = builder.start();
String output = IOUtils.toString(p.getInputStream(), Charset.defaultCharset());
p.waitFor();
if (p.exitValue() != 0) {
String sb = "The exit value of the process is " + p.exitValue()
+ ". Command: " + Joiner.on(" ").join(command) + "\n"
+ "output:\n" + output;
throw new RuntimeException(sb);
}
return output;
}
/**
* Start all Ray processes on this node.
*
* @param isHead Whether this node is the head node. If true, redis server will be started.
*/
public void startRayProcesses(boolean isHead) {
LOGGER.debug("Starting ray runtime @ {}.", rayConfig.nodeIp);
try {
if (isHead) {
startGcs();
}
startObjectStore();
startRaylet(isHead);
LOGGER.info("Ray runtime started @ {}.", rayConfig.nodeIp);
} catch (Exception e) {
// Clean up started processes.
cleanup();
LOGGER.error("Failed to start ray runtime.", e);
throw new RuntimeException("Failed to start ray runtime.", e);
}
}
private void startGcs() {
// start primary redis
String primary = startRedisInstance(rayConfig.nodeIp,
rayConfig.headRedisPort, rayConfig.headRedisPassword, null);
rayConfig.setRedisAddress(primary);
try (Jedis client = new Jedis("127.0.0.1", rayConfig.headRedisPort)) {
if (!Strings.isNullOrEmpty(rayConfig.headRedisPassword)) {
client.auth(rayConfig.headRedisPassword);
}
client.set("UseRaylet", "1");
// Set job counter to compute job id.
client.set("JobCounter", "0");
// Register the number of Redis shards in the primary shard, so that clients
// know how many redis shards to expect under RedisShards.
client.set("NumRedisShards", Integer.toString(rayConfig.numberRedisShards));
// Set session dir for this cluster, so that the drivers which connected to this
// cluster will fetch this session dir as its self's session dir.
client.set("session_dir", rayConfig.getSessionDir());
// start redis shards
for (int i = 0; i < rayConfig.numberRedisShards; i++) {
String shard = startRedisInstance(rayConfig.nodeIp,
rayConfig.redisShardPorts[i], rayConfig.headRedisPassword, i);
client.rpush("RedisShards", shard);
}
}
// start gcs server
String redisPasswordOption = "";
if (!Strings.isNullOrEmpty(rayConfig.headRedisPassword)) {
redisPasswordOption = rayConfig.headRedisPassword;
}
// See `src/ray/gcs/gcs_server/gcs_server_main.cc` for the meaning of each parameter.
final File gcsServerFile = BinaryFileUtil.getNativeFile(
rayConfig.sessionDir, BinaryFileUtil.GCS_SERVER_BINARY_NAME);
Preconditions.checkState(gcsServerFile.setExecutable(true));
List<String> command = ImmutableList.of(
gcsServerFile.getAbsolutePath(),
String.format("--redis_address=%s", rayConfig.getRedisIp()),
String.format("--redis_port=%d", rayConfig.getRedisPort()),
String.format("--config_list=%s",
rayConfig.rayletConfigParameters.entrySet().stream()
.map(entry -> entry.getKey() + "," + entry.getValue()).collect(Collectors
.joining(","))),
String.format("--redis_password=%s", redisPasswordOption)
);
startProcess(command, null, "gcs_server");
}
private String startRedisInstance(String ip, int port, String password, Integer shard) {
final File redisServerFile = BinaryFileUtil.getNativeFile(
rayConfig.sessionDir, BinaryFileUtil.REDIS_SERVER_BINARY_NAME);
Preconditions.checkState(redisServerFile.setExecutable(true));
// The redis module file.
File redisModule = BinaryFileUtil.getNativeFile(
rayConfig.sessionDir, BinaryFileUtil.REDIS_MODULE_LIBRARY_NAME);
Preconditions.checkState(redisModule.setExecutable(true));
List<String> command = Lists.newArrayList(
// The redis-server executable file.
redisServerFile.getAbsolutePath(),
"--protected-mode",
"no",
"--port",
String.valueOf(port),
"--loglevel",
"warning",
"--loadmodule",
// The redis module file.
redisModule.getAbsolutePath()
);
if (!Strings.isNullOrEmpty(password)) {
command.add("--requirepass ");
command.add(password);
}
String name = shard == null ? "redis" : "redis-shard_" + shard;
startProcess(command, null, name);
try (Jedis client = new Jedis("127.0.0.1", port)) {
if (!Strings.isNullOrEmpty(password)) {
client.auth(password);
}
// Configure Redis to only generate notifications for the export keys.
client.configSet("notify-keyspace-events", "Kl");
// Put a time stamp in Redis to indicate when it was started.
client.set("redis_start_time", LocalDateTime.now().toString());
}
return ip + ":" + port;
}
private void startRaylet(boolean isHead) throws IOException {
int hardwareConcurrency = Runtime.getRuntime().availableProcessors();
int maximumStartupConcurrency = Math.max(1,
Math.min(rayConfig.resources.getOrDefault("CPU", 0.0).intValue(), hardwareConcurrency));
String redisPasswordOption = "";
if (!Strings.isNullOrEmpty(rayConfig.headRedisPassword)) {
redisPasswordOption = rayConfig.headRedisPassword;
}
// See `src/ray/raylet/main.cc` for the meaning of each parameter.
final File rayletFile = BinaryFileUtil.getNativeFile(
rayConfig.sessionDir, BinaryFileUtil.RAYLET_BINARY_NAME);
Preconditions.checkState(rayletFile.setExecutable(true));
List<String> command = ImmutableList.of(
rayletFile.getAbsolutePath(),
String.format("--raylet_socket_name=%s", rayConfig.rayletSocketName),
String.format("--store_socket_name=%s", rayConfig.objectStoreSocketName),
String.format("--object_manager_port=%d", 0), // The object manager port.
// The node manager port.
String.format("--node_manager_port=%d", rayConfig.getNodeManagerPort()),
String.format("--node_ip_address=%s", rayConfig.nodeIp),
String.format("--redis_address=%s", rayConfig.getRedisIp()),
String.format("--redis_port=%d", rayConfig.getRedisPort()),
String.format("--num_initial_workers=%d", 0), // number of initial workers
String.format("--maximum_startup_concurrency=%d", maximumStartupConcurrency),
String.format("--static_resource_list=%s",
ResourceUtil.getResourcesStringFromMap(rayConfig.resources)),
String.format("--config_list=%s", rayConfig.rayletConfigParameters.entrySet().stream()
.map(entry -> entry.getKey() + "," + entry.getValue())
.collect(Collectors.joining(","))),
String.format("--python_worker_command=%s", buildPythonWorkerCommand()),
String.format("--java_worker_command=%s", buildWorkerCommand()),
String.format("--redis_password=%s", redisPasswordOption),
isHead ? "--head_node" : ""
);
startProcess(command, null, "raylet");
}
private String concatPath(Stream<String> stream) {
// TODO (hchen): Right now, raylet backend doesn't support worker command with spaces.
// Thus, we have to drop some some paths until that is fixed.
return stream.filter(s -> !s.contains(" ")).collect(Collectors.joining(":"));
}
private String buildWorkerCommand() throws IOException {
List<String> cmd = new ArrayList<>();
cmd.add("java");
cmd.add("-classpath");
// Generate classpath based on current classpath + user-defined classpath.
String classpath = concatPath(Stream.concat(
rayConfig.classpath.stream(),
Stream.of(System.getProperty("java.class.path").split(":"))
));
cmd.add(classpath);
// Write current config to a file, and set the file path as Java worker's config file.
// This allows users to set worker config by setting driver's system properties.
File workerConfigFile = new File(rayConfig.sessionDir + "/java_worker.conf");
FileUtils.write(workerConfigFile, rayConfig.render(), Charset.defaultCharset());
cmd.add("-Dray.config-file=" + workerConfigFile.getAbsolutePath());
if (!rayConfig.codeSearchPath.isEmpty()) {
cmd.add("-Dray.job.code-search-path=" +
String.join(":", rayConfig.codeSearchPath));
}
cmd.add("RAY_WORKER_RAYLET_CONFIG_PLACEHOLDER");
cmd.addAll(rayConfig.jvmParameters);
// jvm options
cmd.add("RAY_WORKER_DYNAMIC_OPTION_PLACEHOLDER");
// Main class
cmd.add(WORKER_CLASS);
String command = Joiner.on(" ").join(cmd);
LOGGER.debug("Worker command is: {}", command);
return command;
}
private void startObjectStore() {
final File objectStoreFile = BinaryFileUtil.getNativeFile(
rayConfig.sessionDir, BinaryFileUtil.PLASMA_STORE_SERVER_BINARY_NAME);
Preconditions.checkState(objectStoreFile.setExecutable(true));
List<String> command = ImmutableList.of(
// The plasma store executable file.
objectStoreFile.getAbsolutePath(),
"-s",
rayConfig.objectStoreSocketName,
"-m",
rayConfig.objectStoreSize.toString()
);
startProcess(command, null, "plasma_store");
}
private String buildPythonWorkerCommand() {
// disable python worker start from raylet, which starts from java
if (rayConfig.pythonWorkerCommand == null) {
return "";
}
List<String> cmd = new ArrayList<>();
cmd.add(rayConfig.pythonWorkerCommand);
cmd.add("--node-ip-address=" + rayConfig.nodeIp);
cmd.add("--object-store-name=" + rayConfig.objectStoreSocketName);
cmd.add("--raylet-name=" + rayConfig.rayletSocketName);
cmd.add("--address=" + rayConfig.getRedisAddress());
String command = cmd.stream().collect(Collectors.joining(" "));
LOGGER.debug("python worker command: {}", command);
return command;
}
}
java/runtime/src/main/java/io/ray/runtime/task/LocalModeTaskSubmitter.java
+5
View File
@@ -240,6 +240,11 @@ public class LocalModeTaskSubmitter implements TaskSubmitter {
placementGroups.remove(id);
}
@Override
public boolean waitPlacementGroupReady(PlacementGroupId id, int timeoutMs) {
return true;
}
@Override
public BaseActorHandle getActor(ActorId actorId) {
return actorHandles.get(actorId).copy();
java/runtime/src/main/java/io/ray/runtime/task/NativeTaskSubmitter.java
+7
View File
@@ -91,6 +91,11 @@ public class NativeTaskSubmitter implements TaskSubmitter {
nativeRemovePlacementGroup(id.getBytes());
}
@Override
public boolean waitPlacementGroupReady(PlacementGroupId id, int timeoutMs) {
return nativeWaitPlacementGroupReady(id.getBytes(), timeoutMs);
}
private static native List<byte[]> nativeSubmitTask(FunctionDescriptor functionDescriptor,
int functionDescriptorHash, List<FunctionArg> args, int numReturns, CallOptions callOptions);
@@ -107,4 +112,6 @@ public class NativeTaskSubmitter implements TaskSubmitter {
private static native void nativeRemovePlacementGroup(byte[] placementGroupId);
private static native boolean nativeWaitPlacementGroupReady(byte[] placementGroupId,
int timeoutMs);
}
java/runtime/src/main/java/io/ray/runtime/task/TaskSubmitter.java
+8
View File
@@ -68,6 +68,14 @@ public interface TaskSubmitter {
*/
void removePlacementGroup(PlacementGroupId id);
/**
* Wait for the placement group to be ready within the specified time.
* @param id Id of placement group.
* @param timeoutMs Timeout in milliseconds.
* @return True if the placement group is created. False otherwise.
*/
boolean waitPlacementGroupReady(PlacementGroupId id, int timeoutMs);
BaseActorHandle getActor(ActorId actorId);
}
java/runtime/src/main/java/io/ray/runtime/util/BinaryFileUtil.java
-9
View File
@@ -12,15 +12,6 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.SystemUtils;
public class BinaryFileUtil {
public static final String REDIS_SERVER_BINARY_NAME = "redis-server";
public static final String GCS_SERVER_BINARY_NAME = "gcs_server";
public static final String PLASMA_STORE_SERVER_BINARY_NAME = "plasma_store_server";
public static final String RAYLET_BINARY_NAME = "raylet";
public static final String REDIS_MODULE_LIBRARY_NAME = "libray_redis_module.so";
public static final String CORE_WORKER_JAVA_LIBRARY = "core_worker_library_java";
java/runtime/src/main/java/io/ray/runtime/util/JniUtils.java
+54 -14
View File
@@ -2,8 +2,9 @@ package io.ray.runtime.util;
import com.google.common.collect.Sets;
import com.sun.jna.NativeLibrary;
import io.ray.runtime.config.RayConfig;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -11,17 +12,7 @@ import org.slf4j.LoggerFactory;
public class JniUtils {
private static final Logger LOGGER = LoggerFactory.getLogger(JniUtils.class);
private static Set<String> loadedLibs = Sets.newHashSet();
/**
* Loads the native library specified by the <code>libraryName</code> argument.
* The <code>libraryName</code> argument must not contain any platform specific
* prefix, file extension or path.
*
* @param libraryName the name of the library.
*/
public static synchronized void loadLibrary(String libraryName) {
loadLibrary(libraryName, false);
}
private static String defaultDestDir;
/**
* Loads the native library specified by the <code>libraryName</code> argument.
@@ -29,15 +20,51 @@ public class JniUtils {
* prefix, file extension or path.
*
* @param libraryName the name of the library.
*/
public static synchronized void loadLibrary(String libraryName) {
loadLibrary(getDefaultDestDir(), libraryName);
}
/**
* Loads the native library specified by the <code>libraryName</code> argument.
* The <code>libraryName</code> argument must not contain any platform specific
* prefix, file extension or path.
*
* @param libraryName the name of the library.
* @param exportSymbols export symbols of library so that it can be used by other libs.
*/
public static synchronized void loadLibrary(String libraryName, boolean exportSymbols) {
loadLibrary(getDefaultDestDir(), libraryName, exportSymbols);
}
/**
* Loads the native library specified by the <code>libraryName</code> argument.
* The <code>libraryName</code> argument must not contain any platform specific
* prefix, file extension or path.
*
* @param destDir The destination dir the library to be extracted.
* @param libraryName the name of the library.
*/
public static synchronized void loadLibrary(String destDir, String libraryName) {
loadLibrary(destDir, libraryName, false);
}
/**
* Loads the native library specified by the <code>libraryName</code> argument.
* The <code>libraryName</code> argument must not contain any platform specific
* prefix, file extension or path.
*
* @param destDir The destination dir the library to be extracted.
* @param libraryName the name of the library.
* @param exportSymbols export symbols of library so that it can be used by other libs.
*/
public static synchronized void loadLibrary(String destDir, String libraryName,
boolean exportSymbols) {
if (!loadedLibs.contains(libraryName)) {
LOGGER.debug("Loading native library {}.", libraryName);
// Load native library.
String fileName = System.mapLibraryName(libraryName);
final String sessionDir = RayConfig.getInstance().sessionDir;
final File file = BinaryFileUtil.getNativeFile(sessionDir, fileName);
final File file = BinaryFileUtil.getNativeFile(destDir, fileName);
if (exportSymbols) {
// Expose library symbols using RTLD_GLOBAL which may be depended by other shared
@@ -50,4 +77,17 @@ public class JniUtils {
}
}
/**
* Cache the result so that multiple calls return the same dest dir.
*/
private static synchronized String getDefaultDestDir() {
if (defaultDestDir == null) {
try {
defaultDestDir = Files.createTempDirectory("native_libs").toString();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return defaultDestDir;
}
}
java/runtime/src/main/resources/ray.default.conf
+4 -33
View File
@@ -18,9 +18,6 @@ ray {
// `CLUSTER`: Ray is running on one or more nodes, with multiple processes.
run-mode: CLUSTER
// Available resources on this node, for example "CPU:4,GPU:0".
resources: ""
// Configuration items about job.
job {
// If worker.mode is DRIVER, specify the job id.
@@ -56,40 +53,12 @@ ray {
max-backup-files: 10
}
// Custom worker jvm parameters.
worker.jvm-parameters: []
// Custom `java.library.path`
// Note, do not use `dir1:dir2` format, put each dir as a list item.
library.path: []
// Custom classpath.
// Note, do not use `dir1:dir2` format, put each dir as a list item.
classpath = []
// ----------------------
// Redis configurations
// ----------------------
redis {
// If `redis.server` isn't provided, which port we should use to start redis server.
// If `head-port` is not provided, it will be generated randomly.
// head-port: 6379
// Below passwords should be consistent with the one defined in python/ray/ray_constants.py.
// The password used to start the redis server on the head node.
head-password: "5241590000000000"
// The password used to connect to the redis server.
password: "5241590000000000"
// If `redis.server` isn't provided, how many Redis shards we should start in addition to the
// primary Redis shard. The ports of these shards will be `head-port + 1`, `head-port + 2`, etc.
shard-number: 1
}
// ----------------------------
// Object store configurations
// ----------------------------
object-store {
// Initial size of the object store.
size: 10 MB
}
// ----------------------------
@@ -97,12 +66,14 @@ ray {
// ----------------------------
raylet {
// See src/ray/ray_config_def.h for options.
// Below section takes effect only if Ray head is started by a driver.
config {
// TODO(zhuohan): enable this for java
put_small_object_in_memory_store: false
}
}
// Whether we enable job manager to submit and manage job.
enable-job-manager: false
// Below args will be appended as parameters of the `ray start` command.
// It takes effect only if Ray head is started by a driver.
head-args: []
}
java/runtime/src/test/java/io/ray/runtime/config/RayConfigTest.java
+25 -21
View File
@@ -2,6 +2,8 @@ package io.ray.runtime.config;
import io.ray.runtime.generated.Common.WorkerType;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -11,37 +13,39 @@ public class RayConfigTest {
@Test
public void testCreateRayConfig() {
Map<String, String> rayletConfig = new HashMap<>();
rayletConfig.put("one", "1");
rayletConfig.put("zero", "0");
rayletConfig.put("positive-integer", "123");
rayletConfig.put("negative-integer", "-123");
rayletConfig.put("float", "-123.456");
rayletConfig.put("true", "true");
rayletConfig.put("false", "false");
rayletConfig.put("string", "abc");
try {
System.setProperty("ray.job.code-search-path", "path/to/ray/job/resource/path");
for (Map.Entry<String, String> entry : rayletConfig.entrySet()) {
System.setProperty("ray.raylet.config." + entry.getKey(), entry.getValue());
}
RayConfig rayConfig = RayConfig.create();
Assert.assertEquals(WorkerType.DRIVER, rayConfig.workerMode);
Assert.assertEquals(Collections.singletonList("path/to/ray/job/resource/path"),
rayConfig.codeSearchPath);
Assert.assertEquals(rayConfig.rayletConfigParameters.get("one"), 1);
Assert.assertEquals(rayConfig.rayletConfigParameters.get("zero"), 0);
Assert.assertEquals(rayConfig.rayletConfigParameters.get("positive-integer"), 123);
Assert.assertEquals(rayConfig.rayletConfigParameters.get("negative-integer"), -123);
Assert.assertEquals(rayConfig.rayletConfigParameters.get("float"), -123.456f);
Assert.assertEquals(rayConfig.rayletConfigParameters.get("true"), true);
Assert.assertEquals(rayConfig.rayletConfigParameters.get("false"), false);
Assert.assertEquals(rayConfig.rayletConfigParameters.get("string"), "abc");
} finally {
// Unset system properties.
System.clearProperty("ray.job.code-search-path");
}
}
@Test
public void testGenerateHeadPortRandomly() {
boolean isSame = true;
final int port1 = RayConfig.create().headRedisPort;
// If we the 2 ports are the same, let's retry.
// This is used to avoid any flaky chance.
for (int i = 0; i < NUM_RETRIES; ++i) {
final int port2 = RayConfig.create().headRedisPort;
if (port1 != port2) {
isSame = false;
break;
for (String key : rayletConfig.keySet()) {
System.clearProperty("ray.raylet.config." + key);
}
}
Assert.assertFalse(isSame);
}
@Test
public void testSpecifyHeadPort() {
System.setProperty("ray.redis.head-port", "11111");
Assert.assertEquals(RayConfig.create().headRedisPort, 11111);
}
}
java/test.sh
+4 -4
View File
@@ -39,13 +39,13 @@ echo "Build test jar."
bazel build //java:all_tests_deploy.jar
# Enable multi-worker feature in Java test
TEST_ARGS=(-Dray.raylet.config.num_workers_per_process_java=10 -Dray.job.num-java-workers-per-process=10)
TEST_ARGS=(-Dray.job.num-java-workers-per-process=10)
echo "Running tests under cluster mode."
# TODO(hchen): Ideally, we should use the following bazel command to run Java tests. However, if there're skipped tests,
# TestNG will exit with code 2. And bazel treats it as test failure.
# bazel test //java:all_tests --action_env=ENABLE_MULTI_LANGUAGE_TESTS=1 --config=ci || cluster_exit_code=$?
ENABLE_MULTI_LANGUAGE_TESTS=1 run_testng java -cp "$ROOT_DIR"/../bazel-bin/java/all_tests_deploy.jar "${TEST_ARGS[@]}" org.testng.TestNG -d /tmp/ray_java_test_output "$ROOT_DIR"/testng.xml
# bazel test //java:all_tests --config=ci || cluster_exit_code=$?
run_testng java -cp "$ROOT_DIR"/../bazel-bin/java/all_tests_deploy.jar "${TEST_ARGS[@]}" org.testng.TestNG -d /tmp/ray_java_test_output "$ROOT_DIR"/testng.xml
echo "Running tests under single-process mode."
# bazel test //java:all_tests --jvmopt="-Dray.run-mode=SINGLE_PROCESS" --config=ci || single_exit_code=$?
@@ -57,7 +57,7 @@ case "${OSTYPE}" in
darwin*) ip=$(ipconfig getifaddr en0);;
*) echo "Can't get ip address for ${OSTYPE}"; exit 1;;
esac
RAY_BACKEND_LOG_LEVEL=debug ray start --head --port=6379 --redis-password=123456 --code-search-path="$PWD/bazel-bin/java/all_tests_deploy.jar"
RAY_BACKEND_LOG_LEVEL=debug ray start --head --port=6379 --redis-password=123456
RAY_BACKEND_LOG_LEVEL=debug java -cp bazel-bin/java/all_tests_deploy.jar -Dray.address="$ip:6379"\
-Dray.redis.password='123456' -Dray.job.code-search-path="$PWD/bazel-bin/java/all_tests_deploy.jar" io.ray.test.MultiDriverTest
ray stop
java/test/src/main/java/io/ray/test/BaseMultiLanguageTest.java
-127
View File
@@ -1,127 +0,0 @@
package io.ray.test;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gson.Gson;
import io.ray.api.Ray;
import io.ray.runtime.config.RayConfig;
import io.ray.runtime.util.NetworkUtil;
import java.io.File;
import java.lang.ProcessBuilder.Redirect;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@Test(groups = {"cluster", "multiLanguage"})
public abstract class BaseMultiLanguageTest {
private static final Logger LOGGER = LoggerFactory.getLogger(BaseMultiLanguageTest.class);
private static final String PLASMA_STORE_SOCKET_NAME = "/tmp/ray/test/plasma_store_socket";
private static final String RAYLET_SOCKET_NAME = "/tmp/ray/test/raylet_socket";
/**
* Execute an external command.
*
* @return Whether the command succeeded.
*/
private boolean executeCommand(List<String> command, int waitTimeoutSeconds,
Map<String, String> env) {
try {
LOGGER.info("Executing command: {}", String.join(" ", command));
ProcessBuilder processBuilder = new ProcessBuilder(command).redirectOutput(Redirect.INHERIT)
.redirectError(Redirect.INHERIT);
for (Entry<String, String> entry : env.entrySet()) {
processBuilder.environment().put(entry.getKey(), entry.getValue());
}
Process process = processBuilder.start();
process.waitFor(waitTimeoutSeconds, TimeUnit.SECONDS);
return process.exitValue() == 0;
} catch (Exception e) {
throw new RuntimeException("Error executing command " + String.join(" ", command), e);
}
}
@BeforeClass(alwaysRun = true, inheritGroups = false)
public void setUp() {
// Delete existing socket files.
for (String socket : ImmutableList.of(RAYLET_SOCKET_NAME, PLASMA_STORE_SOCKET_NAME)) {
File file = new File(socket);
if (file.exists()) {
file.delete();
}
}
String nodeManagerPort = String.valueOf(NetworkUtil.getUnusedPort());
// jars in the `ray` wheel doesn't contains test classes, so we add test classes explicitly.
// Since mvn test classes contains `test` in path and bazel test classes is located at a jar
// with `test` included in the name, we can check classpath `test` to filter out test classes.
List<String> classpath = Stream.of(System.getProperty("java.class.path").split(":"))
.filter(s -> !s.contains(" ") && s.contains("test"))
.collect(Collectors.toList());
// Start ray cluster.
List<String> startCommand = Arrays.asList(
"ray",
"start",
"--head",
"--port=6379",
"--min-worker-port=0",
"--max-worker-port=0",
String.format("--plasma-store-socket-name=%s", PLASMA_STORE_SOCKET_NAME),
String.format("--raylet-socket-name=%s", RAYLET_SOCKET_NAME),
String.format("--node-manager-port=%s", nodeManagerPort),
"--load-code-from-local",
"--system-config=" + new Gson().toJson(RayConfig.create().rayletConfigParameters),
"--code-search-path=" + String.join(":", classpath)
);
if (!executeCommand(startCommand, 10, getRayStartEnv())) {
throw new RuntimeException("Couldn't start ray cluster.");
}
// Connect to the cluster.
Assert.assertFalse(Ray.isInitialized());
System.setProperty("ray.address", "127.0.0.1:6379");
System.setProperty("ray.object-store.socket-name", PLASMA_STORE_SOCKET_NAME);
System.setProperty("ray.raylet.socket-name", RAYLET_SOCKET_NAME);
System.setProperty("ray.raylet.node-manager-port", nodeManagerPort);
Ray.init();
}
/**
* @return The environment variables needed for the `ray start` command.
*/
protected Map<String, String> getRayStartEnv() {
return ImmutableMap.of();
}
@AfterClass(alwaysRun = true, inheritGroups = false)
public void tearDown() {
// Disconnect to the cluster.
Ray.shutdown();
System.clearProperty("ray.address");
System.clearProperty("ray.object-store.socket-name");
System.clearProperty("ray.raylet.socket-name");
System.clearProperty("ray.raylet.node-manager-port");
// Stop ray cluster.
final List<String> stopCommand = ImmutableList.of(
"ray",
"stop"
);
if (!executeCommand(stopCommand, 10, ImmutableMap.of())) {
throw new RuntimeException("Couldn't stop ray cluster");
}
}
}
java/test/src/main/java/io/ray/test/BaseTest.java
-24
View File
@@ -1,46 +1,22 @@
package io.ray.test;
import com.google.common.collect.ImmutableList;
import io.ray.api.Ray;
import java.io.File;
import java.lang.reflect.Method;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
public class BaseTest {
private static final Logger LOGGER = LoggerFactory.getLogger(BaseTest.class);
private List<File> filesToDelete = ImmutableList.of();
@BeforeMethod(alwaysRun = true)
public void setUpBase(Method method) {
Assert.assertFalse(Ray.isInitialized());
Ray.init();
// These files need to be deleted after each test case.
filesToDelete = ImmutableList.of(
new File(Ray.getRuntimeContext().getRayletSocketName()),
new File(Ray.getRuntimeContext().getObjectStoreSocketName()),
// TODO(pcm): This is a workaround for the issue described
// in the PR description of https://github.com/ray-project/ray/pull/5450
// and should be fixed properly.
new File("/tmp/ray/test/raylet_socket")
);
// Make sure the files will be deleted even if the test doesn't exit gracefully.
filesToDelete.forEach(File::deleteOnExit);
}
@AfterMethod(alwaysRun = true)
public void tearDownBase() {
Ray.shutdown();
for (File file : filesToDelete) {
file.delete();
}
}
}
java/test/src/main/java/io/ray/test/CrossLanguageInvocationTest.java
+14 -6
View File
@@ -1,7 +1,6 @@
package io.ray.test;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import io.ray.api.ActorHandle;
import io.ray.api.ObjectRef;
import io.ray.api.PyActorHandle;
@@ -19,17 +18,19 @@ import java.io.InputStream;
import java.math.BigInteger;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
public class CrossLanguageInvocationTest extends BaseMultiLanguageTest {
@Test(groups = {"cluster"})
public class CrossLanguageInvocationTest extends BaseTest {
private static final String PYTHON_MODULE = "test_cross_language_invocation";
@Override
protected Map<String, String> getRayStartEnv() {
@BeforeClass
public void beforeClass() {
// Delete and re-create the temp dir.
File tempDir = new File(
System.getProperty("java.io.tmpdir") + File.separator + "ray_cross_language_test");
@@ -48,7 +49,14 @@ public class CrossLanguageInvocationTest extends BaseMultiLanguageTest {
throw new RuntimeException(e);
}
return ImmutableMap.of("PYTHONPATH", tempDir.getAbsolutePath());
System.setProperty("ray.job.code-search-path",
System.getProperty("java.class.path") + File.pathSeparator
+ tempDir.getAbsolutePath());
}
@AfterClass
public void afterClass() {
System.clearProperty("ray.job.code-search-path");
}
@Test
java/test/src/main/java/io/ray/test/FailureTest.java
+3 -3
View File
@@ -30,13 +30,13 @@ public class FailureTest extends BaseTest {
// This is needed by `testGetThrowsQuicklyWhenFoundException`.
// Set one worker per process. Otherwise, if `badFunc2` and `slowFunc` run in the same
// process, `sleep` will delay `System.exit`.
oldNumWorkersPerProcess = System.getProperty("ray.raylet.config.num_workers_per_process_java");
System.setProperty("ray.raylet.config.num_workers_per_process_java", "1");
oldNumWorkersPerProcess = System.getProperty("ray.job.num-java-workers-per-process");
System.setProperty("ray.job.num-java-workers-per-process", "1");
}
@AfterClass
public void tearDown() {
System.setProperty("ray.raylet.config.num_workers_per_process_java", oldNumWorkersPerProcess);
System.setProperty("ray.job.num-java-workers-per-process", oldNumWorkersPerProcess);
}
public static int badFunc() {
java/test/src/main/java/io/ray/test/GcsClientTest.java
+2 -2
View File
@@ -16,12 +16,12 @@ public class GcsClientTest extends BaseTest {
@BeforeClass
public void setUp() {
System.setProperty("ray.resources", "A:8");
System.setProperty("ray.head-args.0", "--resources={\"A\":8}");
}
@AfterClass
public void tearDown() {
System.clearProperty("ray.resources");
System.clearProperty("ray.head-args.0");
}
public void testGetAllNodeInfo() {
java/test/src/main/java/io/ray/test/GlobalGcTest.java
+2 -2
View File
@@ -17,12 +17,12 @@ public class GlobalGcTest extends BaseTest {
@BeforeClass
public void setUp() {
System.setProperty("ray.object-store.size", "140 MB");
System.setProperty("ray.head-args.0", "--object-store-memory=" + 140L * 1024 * 1024);
}
@AfterClass
public void tearDown() {
System.clearProperty("ray.object-store.size");
System.clearProperty("ray.head-args.0");
}
public static class LargeObjectWithCyclicRef {
java/test/src/main/java/io/ray/test/JobConfigTest.java
-2
View File
@@ -14,7 +14,6 @@ public class JobConfigTest extends BaseTest {
@BeforeClass
public void setupJobConfig() {
System.setProperty("ray.raylet.config.enable_multi_tenancy", "true");
oldNumWorkersPerProcess = System.getProperty("ray.job.num-java-workers-per-process");
System.setProperty("ray.job.num-java-workers-per-process", "3");
System.setProperty("ray.job.jvm-options.0", "-DX=999");
@@ -25,7 +24,6 @@ public class JobConfigTest extends BaseTest {
@AfterClass
public void tearDownJobConfig() {
System.clearProperty("ray.raylet.config.enable_multi_tenancy");
System.setProperty("ray.job.num-java-workers-per-process", oldNumWorkersPerProcess);
System.clearProperty("ray.job.jvm-options.0");
System.clearProperty("ray.job.jvm-options.1");
java/test/src/main/java/io/ray/test/KillActorTest.java
+3 -3
View File
@@ -18,13 +18,13 @@ public class KillActorTest extends BaseTest {
@BeforeClass
public void setUp() {
oldNumWorkersPerProcess = System.getProperty("ray.raylet.config.num_workers_per_process_java");
System.setProperty("ray.raylet.config.num_workers_per_process_java", "1");
oldNumWorkersPerProcess = System.getProperty("ray.job.num-java-workers-per-process");
System.setProperty("ray.job.num-java-workers-per-process", "1");
}
@AfterClass
public void tearDown() {
System.setProperty("ray.raylet.config.num_workers_per_process_java", oldNumWorkersPerProcess);
System.setProperty("ray.job.num-java-workers-per-process", oldNumWorkersPerProcess);
}
public static class HangActor {
java/test/src/main/java/io/ray/test/MultiDriverTest.java
-12
View File
@@ -15,8 +15,6 @@ import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@Test(groups = {"cluster"})
@@ -27,16 +25,6 @@ public class MultiDriverTest extends BaseTest {
private static final int ACTOR_COUNT_PER_DRIVER = 10;
private static final String PID_LIST_PREFIX = "PID: ";
@BeforeClass
public void setUpClass() {
System.setProperty("ray.raylet.config.enable_multi_tenancy", "true");
}
@AfterClass
public void tearDownClass() {
System.clearProperty("ray.raylet.config.enable_multi_tenancy");
}
static int getPid() {
return SystemUtil.pid();
}
java/test/src/main/java/io/ray/test/MultiLanguageClusterTest.java
+2 -1
View File
@@ -5,7 +5,8 @@ import io.ray.api.Ray;
import org.testng.Assert;
import org.testng.annotations.Test;
public class MultiLanguageClusterTest extends BaseMultiLanguageTest {
@Test(groups = {"cluster"})
public class MultiLanguageClusterTest extends BaseTest {
public static String echo(String word) {
return word;
java/test/src/main/java/io/ray/test/PlacementGroupTest.java
+19 -5
View File
@@ -6,6 +6,7 @@ import io.ray.api.id.ActorId;
import io.ray.api.placementgroup.PlacementGroup;
import io.ray.api.placementgroup.PlacementGroupState;
import io.ray.api.placementgroup.PlacementStrategy;
import io.ray.runtime.exception.RayException;
import io.ray.runtime.placementgroup.PlacementGroupImpl;
import java.util.List;
import org.testng.Assert;
@@ -31,8 +32,10 @@ public class PlacementGroupTest extends BaseTest {
// This test just creates a placement group with one bundle.
// It's not comprehensive to test all placement group test cases.
public void testCreateAndCallActor() {
PlacementGroup placementGroup = PlacementGroupTestUtils.createSimpleGroup();
Assert.assertEquals(((PlacementGroupImpl)placementGroup).getName(),"unnamed_group");
PlacementGroupImpl placementGroup = (PlacementGroupImpl)PlacementGroupTestUtils
.createSimpleGroup();
Assert.assertTrue(placementGroup.wait(10));
Assert.assertEquals(placementGroup.getName(),"unnamed_group");
// Test creating an actor from a constructor.
ActorHandle<Counter> actor = Ray.actor(Counter::new, 1)
@@ -40,7 +43,7 @@ public class PlacementGroupTest extends BaseTest {
Assert.assertNotEquals(actor.getId(), ActorId.NIL);
// Test calling an actor.
Assert.assertEquals(Integer.valueOf(1), actor.task(Counter::getValue).remote().get());
Assert.assertEquals(actor.task(Counter::getValue).remote().get(), Integer.valueOf(1));
}
@Test(groups = {"cluster"})
@@ -52,6 +55,8 @@ public class PlacementGroupTest extends BaseTest {
PlacementGroupImpl secondPlacementGroup = (PlacementGroupImpl)PlacementGroupTestUtils
.createNameSpecifiedSimpleGroup("CPU", 1, PlacementStrategy.PACK,
1.0, "second_placement_group");
Assert.assertTrue(firstPlacementGroup.wait(10));
Assert.assertTrue(secondPlacementGroup.wait(10));
PlacementGroupImpl firstPlacementGroupRes =
(PlacementGroupImpl)Ray.getPlacementGroup((firstPlacementGroup).getId());
@@ -97,6 +102,15 @@ public class PlacementGroupTest extends BaseTest {
PlacementGroupImpl removedPlacementGroup =
(PlacementGroupImpl)Ray.getPlacementGroup((secondPlacementGroup).getId());
Assert.assertEquals(removedPlacementGroup.getState(), PlacementGroupState.REMOVED);
// Wait for placement group after it is removed.
int exceptionCount = 0;
try {
removedPlacementGroup.wait(10);
} catch (RayException e) {
++exceptionCount;
}
Assert.assertEquals(exceptionCount, 1);
}
public void testCheckBundleIndex() {
@@ -108,14 +122,14 @@ public class PlacementGroupTest extends BaseTest {
} catch (IllegalArgumentException e) {
++exceptionCount;
}
Assert.assertEquals(1, exceptionCount);
Assert.assertEquals(exceptionCount, 1);
try {
Ray.actor(Counter::new, 1).setPlacementGroup(placementGroup, -1).remote();
} catch (IllegalArgumentException e) {
++exceptionCount;
}
Assert.assertEquals(2, exceptionCount);
Assert.assertEquals(exceptionCount, 2);
}
@Test (expectedExceptions = { IllegalArgumentException.class })
java/test/src/main/java/io/ray/test/RayAlterSuiteListener.java
-3
View File
@@ -18,9 +18,6 @@ public class RayAlterSuiteListener implements IAlterSuiteListener {
XmlGroups groups = new XmlGroups();
XmlRun run = new XmlRun();
run.onExclude(excludedGroup);
if (!"1".equals(System.getenv("ENABLE_MULTI_LANGUAGE_TESTS"))) {
run.onExclude("multiLanguage");
}
groups.setRun(run);
suite.setGroups(groups);
}
java/test/src/main/java/io/ray/test/RaySerializerTest.java
+2 -1
View File
@@ -8,7 +8,8 @@ import io.ray.runtime.object.ObjectSerializer;
import org.testng.Assert;
import org.testng.annotations.Test;
public class RaySerializerTest extends BaseMultiLanguageTest {
@Test(groups = {"cluster"})
public class RaySerializerTest extends BaseTest {
@Test
public void testSerializePyActor() {
java/test/src/main/java/io/ray/test/RayletConfigTest.java
+3 -1
View File
@@ -25,7 +25,9 @@ public class RayletConfigTest extends BaseTest {
public static class TestActor {
public String getConfigValue() {
return TestUtils.getRuntime().getRayConfig().rayletConfigParameters.get(RAY_CONFIG_KEY);
return TestUtils.getRuntime().getRayConfig()
.rayletConfigParameters.get(RAY_CONFIG_KEY)
.toString();
}
}
java/test/src/main/java/io/ray/test/RedisPasswordTest.java
-2
View File
@@ -11,13 +11,11 @@ public class RedisPasswordTest extends BaseTest {
@BeforeClass
public void setUp() {
System.setProperty("ray.redis.head-password", "12345678");
System.setProperty("ray.redis.password", "12345678");
}
@AfterClass
public void tearDown() {
System.clearProperty("ray.redis.head-password");
System.clearProperty("ray.redis.password");
}
java/test/src/main/java/io/ray/test/ReferenceCountingTest.java
+2 -2
View File
@@ -27,12 +27,12 @@ import org.testng.annotations.Test;
public class ReferenceCountingTest extends BaseTest {
@BeforeClass
public void setUp() {
System.setProperty("ray.object-store.size", "100 MB");
System.setProperty("ray.head-args.0", "--object-store-memory=" + 100L * 1024 * 1024);
}
@AfterClass
public void tearDown() {
System.clearProperty("ray.object-store.size");
System.clearProperty("ray.head-args.0");
}
/**
java/test/src/main/java/io/ray/test/ResourcesManagementTest.java
+4 -2
View File
@@ -20,12 +20,14 @@ public class ResourcesManagementTest extends BaseTest {
@BeforeClass
public void setUp() {
System.setProperty("ray.resources", "CPU:4,RES-A:4");
System.setProperty("ray.head-args.0", "--num-cpus=4");
System.setProperty("ray.head-args.1", "--resources={\"RES-A\":4}");
}
@AfterClass
public void tearDown() {
System.clearProperty("ray.resources");
System.clearProperty("ray.head-args.0");
System.clearProperty("ray.head-args.1");
}
public static Integer echo(Integer number) {
java/test/src/main/java/io/ray/test/RuntimeContextTest.java
-12
View File
@@ -14,8 +14,6 @@ import org.testng.annotations.Test;
public class RuntimeContextTest extends BaseTest {
private static JobId JOB_ID = getJobId();
private static String RAYLET_SOCKET_NAME = "/tmp/ray/test/raylet_socket";
private static String OBJECT_STORE_SOCKET_NAME = "/tmp/ray/test/object_store_socket";
private static JobId getJobId() {
// Must be stable across different processes.
@@ -27,23 +25,16 @@ public class RuntimeContextTest extends BaseTest {
@BeforeClass
public void setUp() {
System.setProperty("ray.job.id", JOB_ID.toString());
System.setProperty("ray.raylet.socket-name", RAYLET_SOCKET_NAME);
System.setProperty("ray.object-store.socket-name", OBJECT_STORE_SOCKET_NAME);
}
@AfterClass
public void tearDown() {
System.clearProperty("ray.job.id");
System.clearProperty("ray.raylet.socket-name");
System.clearProperty("ray.object-store.socket-name");
}
@Test
public void testRuntimeContextInDriver() {
Assert.assertEquals(JOB_ID, Ray.getRuntimeContext().getCurrentJobId());
Assert.assertEquals(RAYLET_SOCKET_NAME, Ray.getRuntimeContext().getRayletSocketName());
Assert.assertEquals(OBJECT_STORE_SOCKET_NAME,
Ray.getRuntimeContext().getObjectStoreSocketName());
}
public static class RuntimeContextTester {
@@ -51,9 +42,6 @@ public class RuntimeContextTest extends BaseTest {
public String testRuntimeContext(ActorId actorId) {
Assert.assertEquals(JOB_ID, Ray.getRuntimeContext().getCurrentJobId());
Assert.assertEquals(actorId, Ray.getRuntimeContext().getCurrentActorId());
Assert.assertEquals(RAYLET_SOCKET_NAME, Ray.getRuntimeContext().getRayletSocketName());
Assert.assertEquals(OBJECT_STORE_SOCKET_NAME,
Ray.getRuntimeContext().getObjectStoreSocketName());
return "ok";
}
}
python/ray/__init__.py
+1 -1
View File
@@ -101,7 +101,7 @@ from ray import util # noqa: E402
# Replaced with the current commit when building the wheels.
__commit__ = "{{RAY_COMMIT_SHA}}"
__version__ = "1.1.0.dev0"
__version__ = "1.2.0.dev0"
__all__ = [
"__version__",
python/ray/_private/services.py
+19 -31
View File
@@ -136,7 +136,7 @@ def find_redis_address(address=None):
# --redis_address=123.456.78.910 --node_ip_address=123.456.78.910
# --raylet_socket_name=... --store_socket_name=... --object_manager_port=0
# --min_worker_port=10000 --max_worker_port=10999
# --node_manager_port=58578 --redis_port=6379 --num_initial_workers=8
# --node_manager_port=58578 --redis_port=6379
# --maximum_startup_concurrency=8
# --static_resource_list=node:123.456.78.910,1.0,object_store_memory,66
# --config_list=plasma_store_as_thread,True
@@ -279,7 +279,8 @@ def get_address_info_from_redis_helper(redis_address,
def get_address_info_from_redis(redis_address,
node_ip_address,
num_retries=5,
redis_password=None):
redis_password=None,
no_warning=False):
counter = 0
while True:
try:
@@ -290,10 +291,11 @@ def get_address_info_from_redis(redis_address,
raise
# Some of the information may not be in Redis yet, so wait a little
# bit.
logger.warning(
"Some processes that the driver needs to connect to have "
"not registered with Redis, so retrying. Have you run "
"'ray start' on this node?")
if not no_warning:
logger.warning(
"Some processes that the driver needs to connect to have "
"not registered with Redis, so retrying. Have you run "
"'ray start' on this node?")
time.sleep(1)
counter += 1
@@ -1251,13 +1253,11 @@ def start_raylet(redis_address,
stderr_file=None,
config=None,
java_worker_options=None,
load_code_from_local=False,
huge_pages=False,
fate_share=None,
socket_to_use=None,
head_node=False,
start_initial_python_workers_for_first_job=False,
code_search_path=None):
start_initial_python_workers_for_first_job=False):
"""Start a raylet, which is a combined local scheduler and object manager.
Args:
@@ -1294,9 +1294,6 @@ def start_raylet(redis_address,
config (dict|None): Optional Raylet configuration that will
override defaults in RayConfig.
java_worker_options (list): The command options for Java worker.
code_search_path (list): Code search path for worker. code_search_path
is added to worker command in non-multi-tenancy mode and job_config
in multi-tenancy mode.
Returns:
ProcessInfo for the process that was started.
"""
@@ -1309,7 +1306,6 @@ def start_raylet(redis_address,
raise ValueError("Cannot use valgrind and profiler at the same time.")
assert resource_spec.resolved()
num_initial_workers = resource_spec.num_cpus
static_resources = resource_spec.to_resource_dict()
# Limit the number of workers that can be started in parallel by the
@@ -1346,7 +1342,6 @@ def start_raylet(redis_address,
raylet_name,
redis_password,
session_dir,
code_search_path,
)
else:
java_worker_command = []
@@ -1366,15 +1361,18 @@ def start_raylet(redis_address,
# Create the command that the Raylet will use to start workers.
start_worker_command = [
sys.executable, worker_path, f"--node-ip-address={node_ip_address}",
sys.executable,
worker_path,
f"--node-ip-address={node_ip_address}",
f"--node-manager-port={node_manager_port}",
f"--object-store-name={plasma_store_name}",
f"--raylet-name={raylet_name}", f"--redis-address={redis_address}",
f"--config-list={config_str}", f"--temp-dir={temp_dir}",
f"--metrics-agent-port={metrics_agent_port}"
f"--raylet-name={raylet_name}",
f"--redis-address={redis_address}",
f"--config-list={config_str}",
f"--temp-dir={temp_dir}",
f"--metrics-agent-port={metrics_agent_port}",
"RAY_WORKER_DYNAMIC_OPTION_PLACEHOLDER",
]
if code_search_path:
start_worker_command.append(f"--code-search-path={code_search_path}")
if redis_password:
start_worker_command += [f"--redis-password={redis_password}"]
@@ -1389,12 +1387,6 @@ def start_raylet(redis_address,
if max_worker_port is None:
max_worker_port = 0
if code_search_path is not None and len(code_search_path) > 0:
load_code_from_local = True
if load_code_from_local:
start_worker_command += ["--load-code-from-local"]
# Create agent command
agent_command = [
sys.executable,
@@ -1425,7 +1417,6 @@ def start_raylet(redis_address,
f"--node_ip_address={node_ip_address}",
f"--redis_address={gcs_ip_address}",
f"--redis_port={gcs_port}",
f"--num_initial_workers={num_initial_workers}",
f"--maximum_startup_concurrency={maximum_startup_concurrency}",
f"--static_resource_list={resource_argument}",
f"--config_list={config_str}",
@@ -1485,8 +1476,7 @@ def get_ray_jars_dir():
def build_java_worker_command(java_worker_options, redis_address,
node_manager_port, plasma_store_name,
raylet_name, redis_password, session_dir,
code_search_path):
raylet_name, redis_password, session_dir):
"""This method assembles the command used to start a Java worker.
Args:
@@ -1497,7 +1487,6 @@ def build_java_worker_command(java_worker_options, redis_address,
raylet_name (str): The name of the raylet socket to create.
redis_password (str): The password of connect to redis.
session_dir (str): The path of this session.
code_search_path (list): Teh job code search path.
Returns:
The command string for starting Java worker.
"""
@@ -1518,7 +1507,6 @@ def build_java_worker_command(java_worker_options, redis_address,
pairs.append(("ray.home", RAY_HOME))
pairs.append(("ray.logging.dir", os.path.join(session_dir, "logs")))
pairs.append(("ray.session-dir", session_dir))
pairs.append(("ray.job.code-search-path", code_search_path))
command = ["java"] + ["-D{}={}".format(*pair) for pair in pairs]
command += ["RAY_WORKER_RAYLET_CONFIG_PLACEHOLDER"]
python/ray/_raylet.pyx
+79 -18
View File
@@ -336,6 +336,7 @@ cdef execute_task(
const c_vector[shared_ptr[CRayObject]] &c_args,
const c_vector[CObjectID] &c_arg_reference_ids,
const c_vector[CObjectID] &c_return_ids,
const c_string debugger_breakpoint,
c_vector[shared_ptr[CRayObject]] *returns):
worker = ray.worker.global_worker
@@ -351,6 +352,18 @@ cdef execute_task(
# Automatically restrict the GPUs available to this task.
ray.utils.set_cuda_visible_devices(ray.get_gpu_ids())
# Helper method used to exit current asyncio actor.
# This is called when a KeyboardInterrupt is received by the main thread.
# Upon receiving a KeyboardInterrupt signal, Ray will exit the current
# worker. If the worker is processing normal tasks, Ray treat it as task
# cancellation from ray.cancel(object_ref). If the worker is an asyncio
# actor, Ray will exit the actor.
def exit_current_actor_if_asyncio():
if core_worker.current_actor_is_asyncio():
error = SystemExit(0)
error.is_ray_terminate = True
raise error
function_descriptor = CFunctionDescriptorToPython(
ray_function.GetFunctionDescriptor())
@@ -457,9 +470,26 @@ cdef execute_task(
task_exception = True
try:
with ray.worker._changeproctitle(title, next_title):
if debugger_breakpoint != b"":
ray.util.pdb.set_trace(
breakpoint_uuid=debugger_breakpoint)
outputs = function_executor(*args, **kwargs)
next_breakpoint = (
ray.worker.global_worker.debugger_breakpoint)
if next_breakpoint != b"":
# If this happens, the user typed "remote" and
# there were no more remote calls left in this
# task. In that case we just exit the debugger.
ray.experimental.internal_kv._internal_kv_put(
"RAY_PDB_{}".format(next_breakpoint),
"{\"exit_debugger\": true}")
ray.experimental.internal_kv._internal_kv_del(
"RAY_PDB_CONTINUE_{}".format(next_breakpoint)
)
ray.worker.global_worker.debugger_breakpoint = b""
task_exception = False
except KeyboardInterrupt as e:
exit_current_actor_if_asyncio()
raise TaskCancelledError(
core_worker.get_current_task_id())
if c_return_ids.size() == 1:
@@ -467,6 +497,7 @@ cdef execute_task(
# Check for a cancellation that was called when the function
# was exiting and was raised after the except block.
if not check_signals().ok():
exit_current_actor_if_asyncio()
task_exception = True
raise TaskCancelledError(
core_worker.get_current_task_id())
@@ -523,6 +554,7 @@ cdef CRayStatus task_execution_handler(
const c_vector[shared_ptr[CRayObject]] &c_args,
const c_vector[CObjectID] &c_arg_reference_ids,
const c_vector[CObjectID] &c_return_ids,
const c_string debugger_breakpoint,
c_vector[shared_ptr[CRayObject]] *returns) nogil:
with gil:
@@ -532,7 +564,7 @@ cdef CRayStatus task_execution_handler(
# it does, that indicates that there was an internal error.
execute_task(task_type, task_name, ray_function, c_resources,
c_args, c_arg_reference_ids, c_return_ids,
returns)
debugger_breakpoint, returns)
except Exception:
traceback_str = traceback.format_exc() + (
"An unexpected internal error occurred while the worker "
@@ -1041,6 +1073,7 @@ cdef class CoreWorker:
PlacementGroupID placement_group_id,
int64_t placement_group_bundle_index,
c_bool placement_group_capture_child_tasks,
c_string debugger_breakpoint,
override_environment_variables):
cdef:
unordered_map[c_string, double] c_resources
@@ -1059,15 +1092,18 @@ cdef class CoreWorker:
language.lang, function_descriptor.descriptor)
prepare_args(self, language, args, &args_vector)
with nogil:
CCoreWorkerProcess.GetCoreWorker().SubmitTask(
ray_function, args_vector, CTaskOptions(
name, num_returns, c_resources,
c_override_environment_variables),
&return_ids, max_retries,
c_pair[CPlacementGroupID, int64_t](
c_placement_group_id, placement_group_bundle_index),
placement_group_capture_child_tasks)
# NOTE(edoakes): releasing the GIL while calling this method causes
# segfaults. See relevant issue for details:
# https://github.com/ray-project/ray/pull/12803
CCoreWorkerProcess.GetCoreWorker().SubmitTask(
ray_function, args_vector, CTaskOptions(
name, num_returns, c_resources,
c_override_environment_variables),
&return_ids, max_retries,
c_pair[CPlacementGroupID, int64_t](
c_placement_group_id, placement_group_bundle_index),
placement_group_capture_child_tasks,
debugger_breakpoint)
return VectorToObjectRefs(return_ids)
@@ -1170,6 +1206,21 @@ cdef class CoreWorker:
CCoreWorkerProcess.GetCoreWorker().
RemovePlacementGroup(c_placement_group_id))
def wait_placement_group_ready(self,
PlacementGroupID placement_group_id,
int32_t timeout_seconds):
cdef CRayStatus status
cdef CPlacementGroupID cplacement_group_id = (
CPlacementGroupID.FromBinary(placement_group_id.binary()))
cdef int ctimeout_seconds = timeout_seconds
with nogil:
status = CCoreWorkerProcess.GetCoreWorker() \
.WaitPlacementGroupReady(cplacement_group_id, ctimeout_seconds)
if status.IsNotFound():
raise Exception("Placement group {} does not exist.".format(
placement_group_id))
return status.ok()
def submit_actor_task(self,
Language language,
ActorID actor_id,
@@ -1193,12 +1244,14 @@ cdef class CoreWorker:
language.lang, function_descriptor.descriptor)
prepare_args(self, language, args, &args_vector)
with nogil:
CCoreWorkerProcess.GetCoreWorker().SubmitActorTask(
c_actor_id,
ray_function,
args_vector, CTaskOptions(name, num_returns, c_resources),
&return_ids)
# NOTE(edoakes): releasing the GIL while calling this method causes
# segfaults. See relevant issue for details:
# https://github.com/ray-project/ray/pull/12803
CCoreWorkerProcess.GetCoreWorker().SubmitActorTask(
c_actor_id,
ray_function,
args_vector, CTaskOptions(name, num_returns, c_resources),
&return_ids)
return VectorToObjectRefs(return_ids)
@@ -1400,8 +1453,16 @@ cdef class CoreWorker:
context = worker.get_serialization_context()
serialized_object = context.serialize(output)
data_sizes.push_back(serialized_object.total_bytes)
metadatas.push_back(
string_to_buffer(serialized_object.metadata))
metadata = serialized_object.metadata
if ray.worker.global_worker.debugger_get_breakpoint:
breakpoint = (
ray.worker.global_worker.debugger_get_breakpoint)
metadata += (
b"," + ray_constants.OBJECT_METADATA_DEBUG_PREFIX +
breakpoint.encode())
# Reset debugging context of this worker.
ray.worker.global_worker.debugger_get_breakpoint = b""
metadatas.push_back(string_to_buffer(metadata))
serialized_objects.append(serialized_object)
contained_ids.push_back(
ObjectRefsToVector(serialized_object.contained_object_refs)

Some files were not shown because too many files have changed in this diff Show More