wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-27 20:53:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

[1.0] Remove args from ray start (#10659)

Browse Source 
Resolved Conflicts:
        java/test.sh
        python/ray/tests/test_multi_node.py
This commit is contained in:
Alex Wu
2020-09-18 16:41:23 -07:00
committed by Barak Michener
parent 4b58557309
commit 5bbfc548c1
7 changed files with 45 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cpp/src/ray/util/process_helper.cc
+1 -1
View File
@@ -80,4 +80,4 @@ void ProcessHelper::RayStop(std::shared_ptr<RayConfig> config) {
}
} // namespace api
} // namespace ray
} // namespace ray
java/runtime/src/main/java/io/ray/runtime/runner/RunManager.java
+1 -1
View File
@@ -402,7 +402,7 @@ public class RunManager {
cmd.add("--node-ip-address=" + rayConfig.nodeIp);
cmd.add("--object-store-name=" + rayConfig.objectStoreSocketName);
cmd.add("--raylet-name=" + rayConfig.rayletSocketName);
cmd.add("--redis-address=" + rayConfig.getRedisAddress());
cmd.add("--address=" + rayConfig.getRedisAddress());
String command = cmd.stream().collect(Collectors.joining(" "));
LOGGER.debug("python worker command: {}", command);
java/test.sh
+1 -1
View File
@@ -56,7 +56,7 @@ case "${OSTYPE}" in
darwin*) ip=$(ipconfig getifaddr en0);;
*) echo "Can't get ip address for ${OSTYPE}"; exit 1;;
esac
RAY_BACKEND_LOG_LEVEL=debug ray start --head --redis-port=6379 --redis-password=123456 --code-search-path="$PWD/bazel-bin/java/all_tests_deploy.jar"
RAY_BACKEND_LOG_LEVEL=debug ray start --head --port=6379 --redis-password=123456 --code-search-path="$PWD/bazel-bin/java/all_tests_deploy.jar"
RAY_BACKEND_LOG_LEVEL=debug java -cp bazel-bin/java/all_tests_deploy.jar -Dray.redis.address="$ip:6379"\
-Dray.redis.password='123456' -Dray.job.code-search-path="$PWD/bazel-bin/java/all_tests_deploy.jar" io.ray.test.MultiDriverTest
ray stop
java/test/src/main/java/io/ray/test/BaseMultiLanguageTest.java
+1 -1
View File
@@ -75,7 +75,7 @@ public abstract class BaseMultiLanguageTest {
"ray",
"start",
"--head",
"--redis-port=6379",
"--port=6379",
"--min-worker-port=0",
"--max-worker-port=0",
String.format("--plasma-store-socket-name=%s", PLASMA_STORE_SOCKET_NAME),
python/ray/scripts/scripts.py
+30 -144
View File
@@ -156,16 +156,8 @@ def dashboard(cluster_config_file, cluster_name, port, remote_port):
required=False,
type=str,
help="the IP address of this node")
@click.option(
"--redis-address", required=False, type=str, help="same as --address")
@click.option(
"--address", required=False, type=str, help="the address to use for Ray")
@click.option(
"--redis-port",
required=False,
type=str,
help="(DEPRECATED) the port to use for starting redis. "
"Please use --port instead now.")
@click.option(
"--port",
required=False,
@@ -173,21 +165,10 @@ def dashboard(cluster_config_file, cluster_name, port, remote_port):
help="the port of the head ray process. If not provided, tries to use "
"{0}, falling back to a random port if {0} is "
"not available".format(ray_constants.DEFAULT_PORT))
@click.option(
"--num-redis-shards",
required=False,
type=int,
help=("the number of additional Redis shards to use in "
"addition to the primary Redis shard"))
@click.option(
"--redis-max-clients",
required=False,
type=int,
help=("If provided, attempt to configure Redis with this "
"maximum number of clients."))
@click.option(
"--redis-password",
required=False,
hidden=True,
type=str,
default=ray_constants.REDIS_DEFAULT_PASSWORD,
help="If provided, secure Redis ports with this password")
@@ -229,6 +210,7 @@ def dashboard(cluster_config_file, cluster_name, port, remote_port):
@click.option(
"--memory",
required=False,
hidden=True,
type=int,
help="The amount of memory (in bytes) to make available to workers. "
"By default, this is set to the available memory on the node.")
@@ -241,6 +223,7 @@ def dashboard(cluster_config_file, cluster_name, port, remote_port):
@click.option(
"--redis-max-memory",
required=False,
hidden=True,
type=int,
help="The max amount of memory (in bytes) to allow redis to use. Once the "
"limit is exceeded, redis will start LRU eviction of entries. This only "
@@ -268,20 +251,6 @@ def dashboard(cluster_config_file, cluster_name, port, remote_port):
is_flag=True,
default=False,
help="provide this argument for the head node")
@click.option(
"--include-webui",
default=None,
type=bool,
help="provide this argument if the UI should be started "
"(DEPRECATED: please use --include-dashboard.")
@click.option(
"--webui-host",
required=False,
default="localhost",
help="the host to bind the dashboard server to, either localhost "
"(127.0.0.1) or 0.0.0.0 (available from all interfaces). By default,"
" this is localhost."
" (DEPRECATED: please use --dashboard-host)")
@click.option(
"--include-dashboard",
default=None,
@@ -311,11 +280,6 @@ def dashboard(cluster_config_file, cluster_name, port, remote_port):
required=False,
type=str,
help="object store directory for memory mapped files")
@click.option(
"--huge-pages",
is_flag=True,
default=False,
help="enable support for huge pages in the object store")
@click.option(
"--autoscaling-config",
required=False,
@@ -341,17 +305,20 @@ def dashboard(cluster_config_file, cluster_name, port, remote_port):
help="manually specify the socket path of the raylet process")
@click.option(
"--temp-dir",
hidden=True,
default=None,
help="manually specify the root temporary dir of the Ray process")
@click.option(
"--java-worker-options",
required=False,
hidden=True,
default=None,
type=str,
help="Overwrite the options to start Java workers.")
@click.option(
"--code-search-path",
default=None,
hidden=True,
type=str,
help="A list of directories or jar files separated by colon that specify "
"the search path for user code. This will be used as `CLASSPATH` in "
@@ -359,6 +326,7 @@ def dashboard(cluster_config_file, cluster_name, port, remote_port):
@click.option(
"--system-config",
default=None,
hidden=True,
type=json.loads,
help="Override system configuration defaults.")
@click.option(
@@ -369,33 +337,35 @@ def dashboard(cluster_config_file, cluster_name, port, remote_port):
@click.option(
"--lru-evict",
is_flag=True,
hidden=True,
default=False,
help="Specify whether LRU evict will be used for this cluster.")
@click.option(
"--enable-object-reconstruction",
is_flag=True,
default=False,
hidden=True,
help="Specify whether object reconstruction will be used for this cluster."
)
@click.option(
"--metrics-export-port",
type=int,
hidden=True,
default=None,
help="the port to use to expose Ray metrics through a "
"Prometheus endpoint.")
@add_click_options(logging_options)
def start(node_ip_address, redis_address, address, redis_port, port,
num_redis_shards, redis_max_clients, redis_password,
redis_shard_ports, object_manager_port, node_manager_port,
gcs_server_port, min_worker_port, max_worker_port, memory,
object_store_memory, redis_max_memory, num_cpus, num_gpus, resources,
head, include_webui, webui_host, include_dashboard, dashboard_host,
dashboard_port, block, plasma_directory, huge_pages,
autoscaling_config, no_redirect_worker_output, no_redirect_output,
plasma_store_socket_name, raylet_socket_name, temp_dir,
java_worker_options, code_search_path, load_code_from_local,
system_config, lru_evict, enable_object_reconstruction,
metrics_export_port, log_style, log_color, verbose):
def start(node_ip_address, address, port, redis_password, redis_shard_ports,
object_manager_port, node_manager_port, gcs_server_port,
min_worker_port, max_worker_port, memory, object_store_memory,
redis_max_memory, num_cpus, num_gpus, resources, head,
include_dashboard, dashboard_host, dashboard_port, block,
plasma_directory, autoscaling_config, no_redirect_worker_output,
no_redirect_output, plasma_store_socket_name, raylet_socket_name,
temp_dir, java_worker_options, load_code_from_local,
code_search_path, system_config, lru_evict,
enable_object_reconstruction, metrics_export_port, log_style,
log_color, verbose):
"""Start Ray processes manually on the local machine."""
cli_logger.configure(log_style, log_color, verbose)
@@ -403,59 +373,11 @@ def start(node_ip_address, redis_address, address, redis_port, port,
raise ValueError(
"gcs_server_port can be only assigned when you specify --head.")
if redis_address is not None:
cli_logger.abort("{} is deprecated. Use {} instead.",
cf.bold("--redis-address"), cf.bold("--address"))
raise DeprecationWarning("The --redis-address argument is "
"deprecated. Please use --address instead.")
if redis_port is not None:
cli_logger.warning("{} is being deprecated. Use {} instead.",
cf.bold("--redis-port"), cf.bold("--port"))
cli_logger.old_warning(
logger, "The --redis-port argument will be deprecated soon. "
"Please use --port instead.")
if port is not None and port != redis_port:
cli_logger.abort(
"Incompatible values for {} and {}. Use only {} instead.",
cf.bold("--port"), cf.bold("--redis-port"), cf.bold("--port"))
raise ValueError("Cannot specify both --port and --redis-port "
"as port is a rename of deprecated redis-port")
if include_webui is not None:
cli_logger.warning("{} is being deprecated. Use {} instead.",
cf.bold("--include-webui"),
cf.bold("--include-dashboard"))
cli_logger.old_warning(
logger, "The --include-webui argument will be deprecated soon"
"Please use --include-dashboard instead.")
if include_dashboard is not None:
include_dashboard = include_webui
dashboard_host_default = "localhost"
if webui_host != dashboard_host_default:
cli_logger.warning("{} is being deprecated. Use {} instead.",
cf.bold("--webui-host"),
cf.bold("--dashboard-host"))
cli_logger.old_warning(
logger, "The --webui-host argument will be deprecated"
" soon. Please use --dashboard-host instead.")
if webui_host != dashboard_host and dashboard_host != "localhost":
cli_logger.abort(
"Incompatible values for {} and {}. Use only {} instead.",
cf.bold("--dashboard-host"), cf.bold("--webui-host"),
cf.bold("--dashboard-host"))
raise ValueError(
"Cannot specify both --webui-host and --dashboard-host,"
" please specify only the latter")
else:
dashboard_host = webui_host
# Convert hostnames to numerical IP address.
if node_ip_address is not None:
node_ip_address = services.address_to_ip(node_ip_address)
redis_address = None
if address is not None:
(redis_address, redis_address_ip,
redis_address_port) = services.validate_redis_address(address)
@@ -493,7 +415,7 @@ def start(node_ip_address, redis_address, address, redis_port, port,
num_gpus=num_gpus,
resources=resources,
plasma_directory=plasma_directory,
huge_pages=huge_pages,
huge_pages=False,
plasma_store_socket_name=plasma_store_socket_name,
raylet_socket_name=raylet_socket_name,
temp_dir=temp_dir,
@@ -508,30 +430,13 @@ def start(node_ip_address, redis_address, address, redis_port, port,
enable_object_reconstruction=enable_object_reconstruction,
metrics_export_port=metrics_export_port)
if head:
num_redis_shards = None
# Start Ray on the head node.
if redis_shard_ports is not None:
redis_shard_ports = redis_shard_ports.split(",")
# Infer the number of Redis shards from the ports if the number is
# not provided.
if num_redis_shards is None:
num_redis_shards = len(redis_shard_ports)
# Check that the arguments match.
if len(redis_shard_ports) != num_redis_shards:
cli_logger.error(
"`{}` must be a comma-separated list of ports, "
"with length equal to `{}` (which defaults to {})",
cf.bold("--redis-shard-ports"),
cf.bold("--num-redis-shards"), cf.bold("1"))
cli_logger.abort(
"Example: `{}`",
cf.bold("--num-redis-shards 3 "
"--redis_shard_ports 6380,6381,6382"))
raise Exception("If --redis-shard-ports is provided, it must "
"have the form '6380,6381,6382', and the "
"number of ports provided must equal "
"--num-redis-shards (which is 1 if not "
"provided)")
num_redis_shards = len(redis_shard_ports)
if redis_address is not None:
cli_logger.abort(
@@ -549,11 +454,11 @@ def start(node_ip_address, redis_address, address, redis_port, port,
cli_logger.old_info(logger, "Using IP address {} for this node.",
ray_params.node_ip_address)
ray_params.update_if_absent(
redis_port=port or redis_port,
redis_port=port,
redis_shard_ports=redis_shard_ports,
redis_max_memory=redis_max_memory,
num_redis_shards=num_redis_shards,
redis_max_clients=redis_max_clients,
redis_max_clients=None,
autoscaling_config=autoscaling_config,
)
@@ -616,10 +521,9 @@ def start(node_ip_address, redis_address, address, redis_port, port,
if redis_password else ""))
else:
# Start Ray on a non-head node.
if not (redis_port is None and port is None):
cli_logger.abort("`{}/{}` should not be specified without `{}`.",
cf.bold("--port"), cf.bold("--redis-port"),
cf.bold("--head"))
if not (port is None):
cli_logger.abort("`{}` should not be specified without `{}`.",
cf.bold("--port"), cf.bold("--head"))
raise Exception(
"If --head is not passed in, --port and --redis-port are not "
@@ -636,24 +540,6 @@ def start(node_ip_address, redis_address, address, redis_port, port,
raise Exception("If --head is not passed in, --address must "
"be provided.")
if num_redis_shards is not None:
cli_logger.abort("`{}` should not be specified without `{}`.",
cf.bold("--num-redis-shards"), cf.bold("--head"))
raise Exception("If --head is not passed in, --num-redis-shards "
"must not be provided.")
if redis_max_clients is not None:
cli_logger.abort("`{}` should not be specified without `{}`.",
cf.bold("--redis-max-clients"), cf.bold("--head"))
raise Exception("If --head is not passed in, --redis-max-clients "
"must not be provided.")
if include_webui:
cli_logger.abort("`{}` should not be specified without `{}`.",
cf.bold("--include-web-ui"), cf.bold("--head"))
raise Exception("If --head is not passed in, the --include-webui"
"flag is not relevant.")
if include_dashboard:
cli_logger.abort("`{}` should not be specified without `{}`.",
cf.bold("--include-dashboard"), cf.bold("--head"))
python/ray/tests/test_multi_node.py
+10 -15
View File
@@ -415,23 +415,18 @@ def test_calling_start_ray_head(call_ray_stop_only):
check_call_ray(["start", "--head", "--num-gpus", "100"])
check_call_ray(["stop"])
# Test starting Ray with the max redis clients specified.
check_call_ray(["start", "--head", "--redis-max-clients", "100"])
# Test starting Ray with redis shard ports specified.
check_call_ray(
["start", "--head", "--redis-shard-ports", "6380,6381,6382"])
check_call_ray(["stop"])
if "RAY_USE_NEW_GCS" not in os.environ:
# Test starting Ray with redis shard ports specified.
check_call_ray(
["start", "--head", "--redis-shard-ports", "6380,6381,6382"])
check_call_ray(["stop"])
# Test starting Ray with all arguments specified.
check_call_ray([
"start", "--head", "--redis-shard-ports", "6380,6381,6382",
"--object-manager-port", "12345", "--num-cpus", "2", "--num-gpus",
"0", "--redis-max-clients", "100", "--resources", "{\"Custom\": 1}"
])
check_call_ray(["stop"])
# Test starting Ray with all arguments specified.
check_call_ray([
"start", "--head", "--redis-shard-ports", "6380,6381,6382",
"--object-manager-port", "12345", "--num-cpus", "2", "--num-gpus", "0",
"--resources", "{\"Custom\": 1}"
])
check_call_ray(["stop"])
# Test starting Ray with invalid arguments.
with pytest.raises(subprocess.CalledProcessError):
streaming/java/streaming-api/src/main/java/io/ray/streaming/api/context/ClusterStarter.java
+1 -1
View File
@@ -63,7 +63,7 @@ class ClusterStarter {
"ray",
"start",
"--head",
"--redis-port=6379",
"--port=6379",
String.format("--plasma-store-socket-name=%s", PLASMA_STORE_SOCKET_NAME),
String.format("--raylet-socket-name=%s", RAYLET_SOCKET_NAME),
String.format("--node-manager-port=%s", nodeManagerPort),