wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-30 12:30:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

Lint Python files with Yapf (#1872)

Browse Source
This commit is contained in:
Philipp Moritz
2018-04-11 10:11:35 -07:00
committed by Robert Nishihara
parent a3ddde398c
commit 74162d1492
97 changed files with 3927 additions and 3139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/jenkins_tests/multi_node_docker_test.py
+127 -82
View File
@@ -58,6 +58,7 @@ class DockerRunner(object):
head_container_ip: The IP address of the docker container that runs the
head node.
"""
def __init__(self):
"""Initialize the DockerRunner."""
self.head_container_id = None
@@ -91,11 +92,14 @@ class DockerRunner(object):
Returns:
The IP address of the container.
"""
proc = subprocess.Popen(["docker", "inspect",
"--format={{.NetworkSettings.Networks.bridge"
".IPAddress}}",
container_id],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
proc = subprocess.Popen(
[
"docker", "inspect",
"--format={{.NetworkSettings.Networks.bridge"
".IPAddress}}", container_id
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout_data, _ = wait_for_output(proc)
p = re.compile("([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})")
m = p.match(stdout_data)
@@ -110,23 +114,23 @@ class DockerRunner(object):
"""Start the Ray head node inside a docker container."""
mem_arg = ["--memory=" + mem_size] if mem_size else []
shm_arg = ["--shm-size=" + shm_size] if shm_size else []
volume_arg = (["-v",
"{}:{}".format(os.path.dirname(
os.path.realpath(__file__)),
"/ray/test/jenkins_tests")]
if development_mode else [])
volume_arg = ([
"-v", "{}:{}".format(
os.path.dirname(os.path.realpath(__file__)),
"/ray/test/jenkins_tests")
] if development_mode else [])
command = (["docker", "run", "-d"] + mem_arg + shm_arg + volume_arg +
[docker_image, "ray", "start", "--head", "--block",
"--redis-port=6379",
"--num-redis-shards={}".format(num_redis_shards),
"--num-cpus={}".format(num_cpus),
"--num-gpus={}".format(num_gpus),
"--no-ui"])
command = (["docker", "run", "-d"] + mem_arg + shm_arg + volume_arg + [
docker_image, "ray", "start", "--head", "--block",
"--redis-port=6379",
"--num-redis-shards={}".format(num_redis_shards),
"--num-cpus={}".format(num_cpus), "--num-gpus={}".format(num_gpus),
"--no-ui"
])
print("Starting head node with command:{}".format(command))
proc = subprocess.Popen(command,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
proc = subprocess.Popen(
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout_data, _ = wait_for_output(proc)
container_id = self._get_container_id(stdout_data)
if container_id is None:
@@ -139,29 +143,34 @@ class DockerRunner(object):
"""Start a Ray worker node inside a docker container."""
mem_arg = ["--memory=" + mem_size] if mem_size else []
shm_arg = ["--shm-size=" + shm_size] if shm_size else []
volume_arg = (["-v",
"{}:{}".format(os.path.dirname(
os.path.realpath(__file__)),
"/ray/test/jenkins_tests")]
if development_mode else [])
command = (["docker", "run", "-d"] + mem_arg + shm_arg + volume_arg +
["--shm-size=" + shm_size, docker_image,
"ray", "start", "--block",
"--redis-address={:s}:6379".format(self.head_container_ip),
"--num-cpus={}".format(num_cpus),
"--num-gpus={}".format(num_gpus)])
volume_arg = ([
"-v", "{}:{}".format(
os.path.dirname(os.path.realpath(__file__)),
"/ray/test/jenkins_tests")
] if development_mode else [])
command = (["docker", "run", "-d"] + mem_arg + shm_arg + volume_arg + [
"--shm-size=" + shm_size, docker_image, "ray", "start", "--block",
"--redis-address={:s}:6379".format(self.head_container_ip),
"--num-cpus={}".format(num_cpus), "--num-gpus={}".format(num_gpus)
])
print("Starting worker node with command:{}".format(command))
proc = subprocess.Popen(command, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
proc = subprocess.Popen(
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout_data, _ = wait_for_output(proc)
container_id = self._get_container_id(stdout_data)
if container_id is None:
raise RuntimeError("Failed to find container id")
self.worker_container_ids.append(container_id)
def start_ray(self, docker_image=None, mem_size=None, shm_size=None,
num_nodes=None, num_redis_shards=1, num_cpus=None,
num_gpus=None, development_mode=None):
def start_ray(self,
docker_image=None,
mem_size=None,
shm_size=None,
num_nodes=None,
num_redis_shards=1,
num_cpus=None,
num_gpus=None,
development_mode=None):
"""Start a Ray cluster within docker.
This starts one docker container running the head node and
@@ -200,24 +209,31 @@ class DockerRunner(object):
def _stop_node(self, container_id):
"""Stop a node in the Ray cluster."""
proc = subprocess.Popen(["docker", "kill", container_id],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
proc = subprocess.Popen(
["docker", "kill", container_id],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout_data, _ = wait_for_output(proc)
stopped_container_id = self._get_container_id(stdout_data)
if not container_id == stopped_container_id:
raise Exception("Failed to stop container {}."
.format(container_id))
proc = subprocess.Popen(["docker", "rm", "-f", container_id],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
proc = subprocess.Popen(
["docker", "rm", "-f", container_id],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout_data, _ = wait_for_output(proc)
removed_container_id = self._get_container_id(stdout_data)
if not container_id == removed_container_id:
raise Exception("Failed to remove container {}."
.format(container_id))
print("stop_node", {"container_id": container_id,
"is_head": container_id == self.head_container_id})
print(
"stop_node", {
"container_id": container_id,
"is_head": container_id == self.head_container_id
})
def stop_ray(self):
"""Stop the Ray cluster."""
@@ -236,7 +252,10 @@ class DockerRunner(object):
return success
def run_test(self, test_script, num_drivers, driver_locations=None,
def run_test(self,
test_script,
num_drivers,
driver_locations=None,
timeout_seconds=600):
"""Run a test script.
@@ -258,11 +277,13 @@ class DockerRunner(object):
Raises:
Exception: An exception is raised if the timeout expires.
"""
all_container_ids = ([self.head_container_id] +
self.worker_container_ids)
all_container_ids = (
[self.head_container_id] + self.worker_container_ids)
if driver_locations is None:
driver_locations = [np.random.randint(0, len(all_container_ids))
for _ in range(num_drivers)]
driver_locations = [
np.random.randint(0, len(all_container_ids))
for _ in range(num_drivers)
]
# Define a signal handler and set an alarm to go off in
# timeout_seconds.
@@ -278,13 +299,15 @@ class DockerRunner(object):
for i in range(len(driver_locations)):
# Get the container ID to run the ith driver in.
container_id = all_container_ids[driver_locations[i]]
command = ["docker", "exec", container_id, "/bin/bash", "-c",
("RAY_REDIS_ADDRESS={}:6379 RAY_DRIVER_INDEX={} python "
"{}".format(self.head_container_ip, i, test_script))]
command = [
"docker", "exec", container_id, "/bin/bash", "-c",
("RAY_REDIS_ADDRESS={}:6379 RAY_DRIVER_INDEX={} python "
"{}".format(self.head_container_ip, i, test_script))
]
print("Starting driver with command {}.".format(test_script))
# Start the driver.
p = subprocess.Popen(command, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
p = subprocess.Popen(
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
driver_processes.append(p)
# Wait for the drivers to finish.
@@ -295,8 +318,10 @@ class DockerRunner(object):
print(stdout_data)
print("STDERR:")
print(stderr_data)
results.append({"success": p.returncode == 0,
"return_code": p.returncode})
results.append({
"success": p.returncode == 0,
"return_code": p.returncode
})
# Disable the alarm.
signal.alarm(0)
@@ -307,29 +332,43 @@ class DockerRunner(object):
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Run multinode tests in Docker.")
parser.add_argument("--docker-image", default="ray-project/deploy",
help="docker image")
parser.add_argument(
"--docker-image", default="ray-project/deploy", help="docker image")
parser.add_argument("--mem-size", help="memory size")
parser.add_argument("--shm-size", default="1G", help="shared memory size")
parser.add_argument("--num-nodes", default=1, type=int,
help="number of nodes to use in the cluster")
parser.add_argument("--num-redis-shards", default=1, type=int,
help=("the number of Redis shards to start on the "
"head node"))
parser.add_argument("--num-cpus", type=str,
help=("a comma separated list of values representing "
"the number of CPUs to start each node with"))
parser.add_argument("--num-gpus", type=str,
help=("a comma separated list of values representing "
"the number of GPUs to start each node with"))
parser.add_argument("--num-drivers", default=1, type=int,
help="number of drivers to run")
parser.add_argument("--driver-locations", type=str,
help=("a comma separated list of indices of the "
"containers to run the drivers in"))
parser.add_argument(
"--num-nodes",
default=1,
type=int,
help="number of nodes to use in the cluster")
parser.add_argument(
"--num-redis-shards",
default=1,
type=int,
help=("the number of Redis shards to start on the "
"head node"))
parser.add_argument(
"--num-cpus",
type=str,
help=("a comma separated list of values representing "
"the number of CPUs to start each node with"))
parser.add_argument(
"--num-gpus",
type=str,
help=("a comma separated list of values representing "
"the number of GPUs to start each node with"))
parser.add_argument(
"--num-drivers", default=1, type=int, help="number of drivers to run")
parser.add_argument(
"--driver-locations",
type=str,
help=("a comma separated list of indices of the "
"containers to run the drivers in"))
parser.add_argument("--test-script", required=True, help="test script")
parser.add_argument("--development-mode", action="store_true",
help="use local copies of the test scripts")
parser.add_argument(
"--development-mode",
action="store_true",
help="use local copies of the test scripts")
args = parser.parse_args()
# Parse the number of CPUs and GPUs to use for each worker.
@@ -340,18 +379,24 @@ if __name__ == "__main__":
if args.num_gpus is not None else num_nodes * [0])
# Parse the driver locations.
driver_locations = (None if args.driver_locations is None
else [int(i) for i
in args.driver_locations.split(",")])
driver_locations = (None if args.driver_locations is None else
[int(i) for i in args.driver_locations.split(",")])
d = DockerRunner()
d.start_ray(docker_image=args.docker_image, mem_size=args.mem_size,
shm_size=args.shm_size, num_nodes=num_nodes,
num_redis_shards=args.num_redis_shards, num_cpus=num_cpus,
num_gpus=num_gpus, development_mode=args.development_mode)
d.start_ray(
docker_image=args.docker_image,
mem_size=args.mem_size,
shm_size=args.shm_size,
num_nodes=num_nodes,
num_redis_shards=args.num_redis_shards,
num_cpus=num_cpus,
num_gpus=num_gpus,
development_mode=args.development_mode)
try:
run_results = d.run_test(args.test_script, args.num_drivers,
driver_locations=driver_locations)
run_results = d.run_test(
args.test_script,
args.num_drivers,
driver_locations=driver_locations)
finally:
successfully_stopped = d.stop_ray()