From 1a682e28072e32c73a12a44defac40448a67fc34 Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Fri, 2 Jun 2017 13:17:48 -0700 Subject: [PATCH] Enable starting and stopping ray with "ray start" and "ray stop". (#628) * Install start_ray and stop_ray scripts in setup.py. * Update documentation. * Fix docker tests. * Implement stop_ray script in python. * Fix linting. --- .travis/install-dependencies.sh | 8 +- doc/requirements-doc.txt | 1 + doc/source/install-on-macosx.rst | 2 +- doc/source/install-on-ubuntu.rst | 2 +- .../using-ray-and-docker-on-a-cluster.md | 4 +- doc/source/using-ray-on-a-cluster.rst | 7 +- doc/source/using-ray-on-a-large-cluster.rst | 24 ++- docker/deploy/Dockerfile | 1 - python/ray/scripts/__init__.py | 0 .../ray/scripts/scripts.py | 150 +++++++++++------- python/ray/worker.py | 2 +- python/setup.py | 2 + scripts/start_ray.sh | 5 - scripts/stop_ray.sh | 21 --- test/jenkins_tests/multi_node_docker_test.py | 4 +- test/multi_node_test.py | 46 +++--- 16 files changed, 150 insertions(+), 129 deletions(-) create mode 100644 python/ray/scripts/__init__.py rename scripts/start_ray.py => python/ray/scripts/scripts.py (52%) delete mode 100755 scripts/start_ray.sh delete mode 100755 scripts/stop_ray.sh diff --git a/.travis/install-dependencies.sh b/.travis/install-dependencies.sh index 94c0506f5..13f0fb0ad 100755 --- a/.travis/install-dependencies.sh +++ b/.travis/install-dependencies.sh @@ -20,7 +20,7 @@ fi if [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "linux" ]]; then sudo apt-get update sudo apt-get install -y cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip - sudo pip install cloudpickle funcsigs colorama psutil redis tensorflow flatbuffers + sudo pip install cloudpickle funcsigs click colorama psutil redis tensorflow flatbuffers elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "linux" ]]; then sudo apt-get update sudo apt-get install -y cmake python-dev python-numpy build-essential autoconf curl libtool libboost-all-dev unzip @@ -28,7 +28,7 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "linux" ]]; then wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" - pip install numpy cloudpickle funcsigs colorama psutil redis tensorflow flatbuffers + pip install numpy cloudpickle funcsigs click colorama psutil redis tensorflow flatbuffers elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then # check that brew is installed which -s brew @@ -41,7 +41,7 @@ elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then fi brew install cmake automake autoconf libtool boost sudo easy_install pip - sudo pip install numpy cloudpickle funcsigs colorama psutil redis tensorflow flatbuffers --ignore-installed six + sudo pip install numpy cloudpickle funcsigs click colorama psutil redis tensorflow flatbuffers --ignore-installed six elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then # check that brew is installed which -s brew @@ -57,7 +57,7 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" - pip install numpy cloudpickle funcsigs colorama psutil redis tensorflow flatbuffers + pip install numpy cloudpickle funcsigs click colorama psutil redis tensorflow flatbuffers elif [[ "$LINT" == "1" ]]; then sudo apt-get update sudo apt-get install -y cmake build-essential autoconf curl libtool libboost-all-dev unzip diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt index ab1b03d49..72d1b96d9 100644 --- a/doc/requirements-doc.txt +++ b/doc/requirements-doc.txt @@ -1,4 +1,5 @@ colorama +click cloudpickle funcsigs mock diff --git a/doc/source/install-on-macosx.rst b/doc/source/install-on-macosx.rst index 04148850b..69c91fe1c 100644 --- a/doc/source/install-on-macosx.rst +++ b/doc/source/install-on-macosx.rst @@ -17,7 +17,7 @@ To install Ray, first install the following dependencies. We recommend using brew update brew install cmake automake autoconf libtool boost wget - pip install numpy cloudpickle funcsigs colorama psutil redis flatbuffers --ignore-installed six + pip install numpy cloudpickle funcsigs click colorama psutil redis flatbuffers --ignore-installed six If you are using Anaconda, you may also need to run the following. diff --git a/doc/source/install-on-ubuntu.rst b/doc/source/install-on-ubuntu.rst index 3d1e63d0e..56564a00f 100644 --- a/doc/source/install-on-ubuntu.rst +++ b/doc/source/install-on-ubuntu.rst @@ -21,7 +21,7 @@ To install Ray, first install the following dependencies. We recommend using sudo apt-get install python-dev # For Python 2. sudo apt-get install python3-dev # For Python 3. - pip install numpy cloudpickle funcsigs colorama psutil redis flatbuffers + pip install numpy cloudpickle funcsigs click colorama psutil redis flatbuffers If you are using Anaconda, you may also need to run the following. diff --git a/doc/source/using-ray-and-docker-on-a-cluster.md b/doc/source/using-ray-and-docker-on-a-cluster.md index 117afb963..9ae39d178 100644 --- a/doc/source/using-ray-and-docker-on-a-cluster.md +++ b/doc/source/using-ray-and-docker-on-a-cluster.md @@ -165,7 +165,7 @@ eval $(aws ecr get-login --region ) docker run \ -d --shm-size= --net=host \ \ - /ray/scripts/start_ray.sh --head \ + ray start --head \ --object-manager-port=8076 \ --redis-port=6379 \ --num-workers= @@ -182,7 +182,7 @@ To start Ray on the worker nodes create a script `start-worker-docker.sh` with c eval $(aws ecr get-login --region ) docker run -d --shm-size= --net=host \ \ - /ray/scripts/start_ray.sh \ + ray start \ --object-manager-port=8076 \ --redis-address= \ --num-workers= diff --git a/doc/source/using-ray-on-a-cluster.rst b/doc/source/using-ray-on-a-cluster.rst index 97af9d23c..a25b702e1 100644 --- a/doc/source/using-ray-on-a-cluster.rst +++ b/doc/source/using-ray-on-a-cluster.rst @@ -24,7 +24,7 @@ If the ``--redis-port`` argument is omitted, Ray will choose a port at random. .. code-block:: bash - ./ray/scripts/start_ray.sh --head --redis-port=6379 + ray start --head --redis-port=6379 The command will print out the address of the Redis server that was started (and some other address information). @@ -35,7 +35,7 @@ should look something like ``123.45.67.89:6379``). .. code-block:: bash - ./ray/scripts/start_ray.sh --redis-address= + ray start --redis-address= If you wish to specify that a machine has 10 CPUs and 1 GPU, you can do this with the flags ``--num-cpus=10`` and ``--num-gpus=1``. If these flags are not @@ -77,5 +77,4 @@ following. Stopping Ray ~~~~~~~~~~~~ -When you want to stop the Ray processes, run ``./ray/scripts/stop_ray.sh`` on -each node. +When you want to stop the Ray processes, run ``ray stop`` on each node. diff --git a/doc/source/using-ray-on-a-large-cluster.rst b/doc/source/using-ray-on-a-large-cluster.rst index ee308f386..972bd6a0b 100644 --- a/doc/source/using-ray-on-a-large-cluster.rst +++ b/doc/source/using-ray-on-a-large-cluster.rst @@ -102,7 +102,7 @@ On the head node, run the following: .. code-block:: bash - ./ray/scripts/start_ray.sh --head --redis-port=6379 + ray start --head --redis-port=6379 **Start Ray on the worker nodes** @@ -114,7 +114,7 @@ Create a file ``start_worker.sh`` that contains something like the following: # Make sure the SSH session has the correct version of Python on its path. # You will probably have to change the line below. export PATH=/home/ubuntu/anaconda3/bin/:$PATH - ray/scripts/start_ray.sh --redis-address=:6379 + ray start --redis-address=:6379 This script, when run on the worker nodes, will start up Ray. You will need to replace ```` with the IP address that worker nodes will use to @@ -186,18 +186,30 @@ Stopping Ray **Stop Ray on worker nodes** +Create a file ``stop_worker.sh`` that contains something like the following: + .. code-block:: bash - parallel-ssh -h workers.txt -P ray/scripts/stop_ray.sh + # Make sure the SSH session has the correct version of Python on its path. + # You will probably have to change the line below. + export PATH=/home/ubuntu/anaconda3/bin/:$PATH + ray stop -This command will execute the ``stop_ray.sh`` script on each of the worker -nodes. +This script, when run on the worker nodes, will stop Ray. Note, you will need to +replace ``/home/ubuntu/anaconda3/bin/`` with the correct path to your Python +installation. + +Now use ``parallel-ssh`` to stop Ray on each worker node. + +.. code-block:: bash + + parallel-ssh -h workers.txt -P -I < stop_worker.sh **Stop Ray on the head node** .. code-block:: bash - ray/scripts/stop_ray.sh + ray stop Upgrading Ray ~~~~~~~~~~~~~ diff --git a/docker/deploy/Dockerfile b/docker/deploy/Dockerfile index 543b7ba97..f3d797dee 100644 --- a/docker/deploy/Dockerfile +++ b/docker/deploy/Dockerfile @@ -7,4 +7,3 @@ ADD git-rev /ray/git-rev WORKDIR /ray/python RUN python setup.py install WORKDIR /ray -RUN echo "tail -f /dev/null" >> scripts/start_ray.sh diff --git a/python/ray/scripts/__init__.py b/python/ray/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/start_ray.py b/python/ray/scripts/scripts.py similarity index 52% rename from scripts/start_ray.py rename to python/ray/scripts/scripts.py index 31f9cc5af..bc21e7c32 100644 --- a/scripts/start_ray.py +++ b/python/ray/scripts/scripts.py @@ -2,33 +2,12 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import argparse +import click import redis +import subprocess import ray.services as services -parser = argparse.ArgumentParser( - description="Start the Ray processes on a node.") -parser.add_argument("--node-ip-address", required=False, type=str, - help="the IP address of this node") -parser.add_argument("--redis-address", required=False, type=str, - help="the address to use for connecting to Redis") -parser.add_argument("--redis-port", required=False, type=str, - help="the port to use for starting Redis") -parser.add_argument("--num-redis-shards", required=False, type=int, - help=("the number of additional Redis shards to use in " - "addition to the primary Redis shard")) -parser.add_argument("--object-manager-port", required=False, type=int, - help="the port to use for starting the object manager") -parser.add_argument("--num-workers", required=False, type=int, - help="the initial number of workers to start on this node") -parser.add_argument("--num-cpus", required=False, type=int, - help="the number of CPUs on this node") -parser.add_argument("--num-gpus", required=False, type=int, - help="the number of GPUs on this node") -parser.add_argument("--head", action="store_true", - help="provide this argument for the head node") - def check_no_existing_redis_clients(node_ip_address, redis_address): redis_ip_address, redis_port = redis_address.split(":") @@ -56,48 +35,72 @@ def check_no_existing_redis_clients(node_ip_address, redis_address): "with this IP address.") -if __name__ == "__main__": - args = parser.parse_args() +@click.group() +def cli(): + pass + +@click.command() +@click.option("--node-ip-address", required=False, type=str, + help="the IP address of this node") +@click.option("--redis-address", required=False, type=str, + help="the address to use for connecting to Redis") +@click.option("--redis-port", required=False, type=str, + help="the port to use for starting Redis") +@click.option("--num-redis-shards", required=False, type=int, + help=("the number of additional Redis shards to use in " + "addition to the primary Redis shard")) +@click.option("--object-manager-port", required=False, type=int, + help="the port to use for starting the object manager") +@click.option("--num-workers", required=False, type=int, + help="the initial number of workers to start on this node") +@click.option("--num-cpus", required=False, type=int, + help="the number of CPUs on this node") +@click.option("--num-gpus", required=False, type=int, + help="the number of GPUs on this node") +@click.option("--head", is_flag=True, default=False, + help="provide this argument for the head node") +@click.option("--block", is_flag=True, default=False, + help="provide this argument to block forever in this command") +def start(node_ip_address, redis_address, redis_port, num_redis_shards, + object_manager_port, num_workers, num_cpus, num_gpus, head, block): # Note that we redirect stdout and stderr to /dev/null because otherwise # attempts to print may cause exceptions if a process is started inside of an # SSH connection and the SSH connection dies. TODO(rkn): This is a temporary # fix. We should actually redirect stdout and stderr to Redis in some way. - if args.head: + if head: # Start Ray on the head node. - if args.redis_address is not None: + if redis_address is not None: raise Exception("If --head is passed in, a Redis server will be " "started, so a Redis address should not be provided.") # Get the node IP address if one is not provided. - if args.node_ip_address is None: + if node_ip_address is None: node_ip_address = services.get_node_ip_address() - else: - node_ip_address = args.node_ip_address print("Using IP address {} for this node.".format(node_ip_address)) address_info = {} # Use the provided object manager port if there is one. - if args.object_manager_port is not None: - address_info["object_manager_ports"] = [args.object_manager_port] + if object_manager_port is not None: + address_info["object_manager_ports"] = [object_manager_port] if address_info == {}: address_info = None address_info = services.start_ray_head( address_info=address_info, node_ip_address=node_ip_address, - redis_port=args.redis_port, - num_workers=args.num_workers, + redis_port=redis_port, + num_workers=num_workers, cleanup=False, redirect_output=True, - num_cpus=args.num_cpus, - num_gpus=args.num_gpus, - num_redis_shards=args.num_redis_shards) + num_cpus=num_cpus, + num_gpus=num_gpus, + num_redis_shards=num_redis_shards) print(address_info) print("\nStarted Ray on this node. You can add additional nodes to the " "cluster by calling\n\n" - " ./scripts/start_ray.sh --redis-address {}\n\n" + " ray start --redis-address {}\n\n" "from the node you wish to add. You can connect a driver to the " "cluster from Python by running\n\n" " import ray\n" @@ -105,43 +108,80 @@ if __name__ == "__main__": "If you have trouble connecting from a different machine, check " "that your firewall is configured properly. If you wish to " "terminate the processes that have been started, run\n\n" - " ./scripts/stop_ray.sh".format(address_info["redis_address"], - address_info["redis_address"])) + " ray stop".format(address_info["redis_address"], + address_info["redis_address"])) else: # Start Ray on a non-head node. - if args.redis_port is not None: + if redis_port is not None: raise Exception("If --head is not passed in, --redis-port is not " "allowed") - if args.redis_address is None: + if redis_address is None: raise Exception("If --head is not passed in, --redis-address must be " "provided.") - if args.num_redis_shards is not None: + if num_redis_shards is not None: raise Exception("If --head is not passed in, --num-redis-shards must " "not be provided.") - redis_ip_address, redis_port = args.redis_address.split(":") + redis_ip_address, redis_port = redis_address.split(":") # Wait for the Redis server to be started. And throw an exception if we # can't connect to it. services.wait_for_redis_to_start(redis_ip_address, int(redis_port)) # Get the node IP address if one is not provided. - if args.node_ip_address is None: - node_ip_address = services.get_node_ip_address(args.redis_address) - else: - node_ip_address = args.node_ip_address + if node_ip_address is None: + node_ip_address = services.get_node_ip_address(redis_address) print("Using IP address {} for this node.".format(node_ip_address)) # Check that there aren't already Redis clients with the same IP address # connected with this Redis instance. This raises an exception if the Redis # server already has clients on this node. - check_no_existing_redis_clients(node_ip_address, args.redis_address) + check_no_existing_redis_clients(node_ip_address, redis_address) address_info = services.start_ray_node( node_ip_address=node_ip_address, - redis_address=args.redis_address, - object_manager_ports=[args.object_manager_port], - num_workers=args.num_workers, + redis_address=redis_address, + object_manager_ports=[object_manager_port], + num_workers=num_workers, cleanup=False, redirect_output=True, - num_cpus=args.num_cpus, - num_gpus=args.num_gpus) + num_cpus=num_cpus, + num_gpus=num_gpus) print(address_info) print("\nStarted Ray on this node. If you wish to terminate the processes " "that have been started, run\n\n" - " ./scripts/stop_ray.sh") + " ray stop") + + if block: + import time + while True: + time.sleep(30) + + +@click.command() +def stop(): + subprocess.call(["killall global_scheduler plasma_store plasma_manager " + "local_scheduler"], shell=True) + + # Find the PID of the monitor process and kill it. + subprocess.call(["kill $(ps aux | grep monitor.py | awk '{ print $2 }') " + "2> /dev/null"], shell=True) + + # Find the PID of the Redis process and kill it. + subprocess.call(["kill $(ps aux | grep redis-server | awk '{ print $2 }') " + "2> /dev/null"], shell=True) + + # Find the PIDs of the worker processes and kill them. + subprocess.call(["kill $(ps aux | grep default_worker.py | " + "awk '{ print $2 }') 2> /dev/null"], shell=True) + + # Find the PID of the Ray log monitor process and kill it. + subprocess.call(["kill $(ps aux | grep log_monitor.py | " + "awk '{ print $2 }') 2> /dev/null"], shell=True) + + +cli.add_command(start) +cli.add_command(stop) + + +def main(): + return cli() + + +if __name__ == "__main__": + main() diff --git a/python/ray/worker.py b/python/ray/worker.py index b819208d5..bd353db30 100644 --- a/python/ray/worker.py +++ b/python/ray/worker.py @@ -751,7 +751,7 @@ def get_address_info_from_redis(redis_address, node_ip_address, num_retries=5): # Some of the information may not be in Redis yet, so wait a little bit. print("Some processes that the driver needs to connect to have not " "registered with Redis, so retrying. Have you run " - "./scripts/start_ray.sh on this node?") + "'ray start' on this node?") time.sleep(1) counter += 1 diff --git a/python/setup.py b/python/setup.py index f7b520db5..9f8c49c86 100644 --- a/python/setup.py +++ b/python/setup.py @@ -65,11 +65,13 @@ setup(name="ray", distclass=BinaryDistribution, install_requires=["numpy", "funcsigs", + "click", "colorama", "psutil", "redis", "cloudpickle >= 0.2.2", "flatbuffers"], + entry_points={"console_scripts": ["ray=ray.scripts.scripts:main"]}, include_package_data=True, zip_safe=False, license="Apache 2.0") diff --git a/scripts/start_ray.sh b/scripts/start_ray.sh deleted file mode 100755 index dad992067..000000000 --- a/scripts/start_ray.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) - -python "$ROOT_DIR/start_ray.py" "$@" diff --git a/scripts/stop_ray.sh b/scripts/stop_ray.sh deleted file mode 100755 index bda47f56b..000000000 --- a/scripts/stop_ray.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -killall global_scheduler plasma_store plasma_manager local_scheduler - -# Find the PID of the monitor process and kill it. -kill $(ps aux | grep monitor.py | awk '{ print $2 }') 2> /dev/null - -# Find the PID of the Redis process and kill it. -kill $(ps aux | grep redis-server | awk '{ print $2 }') 2> /dev/null - -# Find the PIDs of the worker processes and kill them. -kill $(ps aux | grep default_worker.py | awk '{ print $2 }') 2> /dev/null - -# Kill the processes related to the web UI. -killall polymer - -# Find the PID of the Ray UI backend process and kill it. -kill $(ps aux | grep ray_ui.py | awk '{ print $2 }') 2> /dev/null - -# Find the PID of the Ray log monitor process and kill it. -kill $(ps aux | grep log_monitor.py | awk '{ print $2 }') 2> /dev/null diff --git a/test/jenkins_tests/multi_node_docker_test.py b/test/jenkins_tests/multi_node_docker_test.py index 5e2c91fc2..39113d893 100644 --- a/test/jenkins_tests/multi_node_docker_test.py +++ b/test/jenkins_tests/multi_node_docker_test.py @@ -112,7 +112,7 @@ class DockerRunner(object): if development_mode else []) command = (["docker", "run", "-d"] + mem_arg + shm_arg + volume_arg + - [docker_image, "/ray/scripts/start_ray.sh", "--head", + [docker_image, "ray", "start", "--head", "--block", "--redis-port=6379", "--num-redis-shards={}".format(num_redis_shards), "--num-cpus={}".format(num_cpus), @@ -139,7 +139,7 @@ class DockerRunner(object): if development_mode else []) command = (["docker", "run", "-d"] + mem_arg + shm_arg + volume_arg + ["--shm-size=" + shm_size, docker_image, - "/ray/scripts/start_ray.sh", + "ray", "start", "--block", "--redis-address={:s}:6379".format(self.head_container_ip), "--num-cpus={}".format(num_cpus), "--num-gpus={}".format(num_gpus)]) diff --git a/test/multi_node_test.py b/test/multi_node_test.py index 307ad6932..31aef4aa3 100644 --- a/test/multi_node_test.py +++ b/test/multi_node_test.py @@ -2,24 +2,18 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import os import unittest import ray import subprocess import tempfile import time -start_ray_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), - "../scripts/start_ray.sh") -stop_ray_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), - "../scripts/stop_ray.sh") - class MultiNodeTest(unittest.TestCase): def setUp(self): # Start the Ray processes on this machine. - out = subprocess.check_output([start_ray_script, "--head"]).decode("ascii") + out = subprocess.check_output(["ray", "start", "--head"]).decode("ascii") # Get the redis address from the output. redis_substring_prefix = "redis_address=\"" redis_address_location = (out.find(redis_substring_prefix) + @@ -29,7 +23,7 @@ class MultiNodeTest(unittest.TestCase): def tearDown(self): # Kill the Ray cluster. - subprocess.Popen([stop_ray_script]).wait() + subprocess.Popen(["ray", "stop"]).wait() def testErrorIsolation(self): # Connect a driver to the Ray cluster. @@ -162,53 +156,53 @@ class StartRayScriptTest(unittest.TestCase): # the non-head node code path. # Test starting Ray with no arguments. - subprocess.check_output([start_ray_script, "--head"]).decode("ascii") - subprocess.Popen([stop_ray_script]).wait() + subprocess.check_output(["ray", "start", "--head"]).decode("ascii") + subprocess.Popen(["ray", "stop"]).wait() # Test starting Ray with a number of workers specified. - subprocess.check_output([start_ray_script, "--head", "--num-workers", + subprocess.check_output(["ray", "start", "--head", "--num-workers", "20"]) - subprocess.Popen([stop_ray_script]).wait() + subprocess.Popen(["ray", "stop"]).wait() # Test starting Ray with a redis port specified. - subprocess.check_output([start_ray_script, "--head", + subprocess.check_output(["ray", "start", "--head", "--redis-port", "6379"]) - subprocess.Popen([stop_ray_script]).wait() + subprocess.Popen(["ray", "stop"]).wait() # Test starting Ray with a node IP address specified. - subprocess.check_output([start_ray_script, "--head", + subprocess.check_output(["ray", "start", "--head", "--node-ip-address", "127.0.0.1"]) - subprocess.Popen([stop_ray_script]).wait() + subprocess.Popen(["ray", "stop"]).wait() # Test starting Ray with an object manager port specified. - subprocess.check_output([start_ray_script, "--head", + subprocess.check_output(["ray", "start", "--head", "--object-manager-port", "12345"]) - subprocess.Popen([stop_ray_script]).wait() + subprocess.Popen(["ray", "stop"]).wait() # Test starting Ray with the number of CPUs specified. - subprocess.check_output([start_ray_script, "--head", + subprocess.check_output(["ray", "start", "--head", "--num-cpus", "100"]) - subprocess.Popen([stop_ray_script]).wait() + subprocess.Popen(["ray", "stop"]).wait() # Test starting Ray with the number of GPUs specified. - subprocess.check_output([start_ray_script, "--head", + subprocess.check_output(["ray", "start", "--head", "--num-gpus", "100"]) - subprocess.Popen([stop_ray_script]).wait() + subprocess.Popen(["ray", "stop"]).wait() # Test starting Ray with all arguments specified. - subprocess.check_output([start_ray_script, "--head", + subprocess.check_output(["ray", "start", "--head", "--num-workers", "20", "--redis-port", "6379", "--object-manager-port", "12345", "--num-cpus", "100", "--num-gpus", "0"]) - subprocess.Popen([stop_ray_script]).wait() + subprocess.Popen(["ray", "stop"]).wait() # Test starting Ray with invalid arguments. with self.assertRaises(Exception): - subprocess.check_output([start_ray_script, "--head", + subprocess.check_output(["ray", "start", "--head", "--redis-address", "127.0.0.1:6379"]) - subprocess.Popen([stop_ray_script]).wait() + subprocess.Popen(["ray", "stop"]).wait() if __name__ == "__main__":