mirror of
https://github.com/wassname/ray.git
synced 2026-07-23 13:10:11 +08:00
Define a Node class to manage Ray processes. (#3733)
* Implement Node class and move most of services.py into it. * Wait for nodes as they are added to the cluster. * Fix Redis authentication bug. * Fix bug in client table ordering. * Address comments. * Kill raylet before plasma store in test. * Minor
This commit is contained in:
Robert Nishihara
committed by
Philipp Moritz
Philipp Moritz
parent
fa2bfa6d76
commit
8723d6b061
+112
-436
@@ -8,13 +8,10 @@ import multiprocessing
|
||||
import os
|
||||
import random
|
||||
import resource
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
import redis
|
||||
|
||||
import pyarrow
|
||||
@@ -22,30 +19,8 @@ import pyarrow
|
||||
import ray.ray_constants as ray_constants
|
||||
import ray.plasma
|
||||
|
||||
from ray.tempfile_services import (
|
||||
get_ipython_notebook_path, get_logs_dir_path, get_raylet_socket_name,
|
||||
get_temp_root, new_log_monitor_log_file, new_monitor_log_file,
|
||||
new_plasma_store_log_file, new_raylet_log_file, new_redis_log_file,
|
||||
new_webui_log_file, set_temp_root)
|
||||
|
||||
PROCESS_TYPE_MONITOR = "monitor"
|
||||
PROCESS_TYPE_LOG_MONITOR = "log_monitor"
|
||||
PROCESS_TYPE_WORKER = "worker"
|
||||
PROCESS_TYPE_RAYLET = "raylet"
|
||||
PROCESS_TYPE_PLASMA_STORE = "plasma_store"
|
||||
PROCESS_TYPE_REDIS_SERVER = "redis_server"
|
||||
PROCESS_TYPE_WEB_UI = "web_ui"
|
||||
|
||||
# This is a dictionary tracking all of the processes of different types that
|
||||
# have been started by this services module. Note that the order of the keys is
|
||||
# important because it determines the order in which these processes will be
|
||||
# terminated when Ray exits, and certain orders will cause errors to be logged
|
||||
# to the screen.
|
||||
all_processes = OrderedDict(
|
||||
[(PROCESS_TYPE_MONITOR, []), (PROCESS_TYPE_LOG_MONITOR, []),
|
||||
(PROCESS_TYPE_WORKER, []), (PROCESS_TYPE_RAYLET, []),
|
||||
(PROCESS_TYPE_PLASMA_STORE, []), (PROCESS_TYPE_REDIS_SERVER, []),
|
||||
(PROCESS_TYPE_WEB_UI, [])], )
|
||||
from ray.tempfile_services import (get_ipython_notebook_path, get_temp_root,
|
||||
new_redis_log_file)
|
||||
|
||||
# True if processes are run in the valgrind profiler.
|
||||
RUN_RAYLET_PROFILER = False
|
||||
@@ -106,85 +81,24 @@ def new_port():
|
||||
return random.randint(10000, 65535)
|
||||
|
||||
|
||||
def kill_process(p):
|
||||
"""Kill a process.
|
||||
def remaining_processes_alive(exclude=None):
|
||||
"""See if the remaining processes are alive or not.
|
||||
|
||||
Args:
|
||||
p: The process to kill.
|
||||
Note that this ignores processes that have been explicitly killed,
|
||||
e.g., via a command like node.kill_raylet().
|
||||
|
||||
Returns:
|
||||
True if the process was killed successfully and false otherwise.
|
||||
True if the remaining processes started by ray.init() are alive and
|
||||
False otherwise.
|
||||
|
||||
Raises:
|
||||
Exception: An exception is raised if the processes were not started by
|
||||
ray.init().
|
||||
"""
|
||||
if p.poll() is not None:
|
||||
# The process has already terminated.
|
||||
return True
|
||||
if any([RUN_RAYLET_PROFILER, RUN_PLASMA_STORE_PROFILER]):
|
||||
# Give process signal to write profiler data.
|
||||
os.kill(p.pid, signal.SIGINT)
|
||||
# Wait for profiling data to be written.
|
||||
time.sleep(0.1)
|
||||
|
||||
# Allow the process one second to exit gracefully.
|
||||
p.terminate()
|
||||
timer = threading.Timer(1, lambda p: p.kill(), [p])
|
||||
try:
|
||||
timer.start()
|
||||
p.wait()
|
||||
finally:
|
||||
timer.cancel()
|
||||
|
||||
if p.poll() is not None:
|
||||
return True
|
||||
|
||||
# If the process did not exit within one second, force kill it.
|
||||
p.kill()
|
||||
if p.poll() is not None:
|
||||
return True
|
||||
|
||||
# The process was not killed for some reason.
|
||||
return False
|
||||
|
||||
|
||||
def cleanup():
|
||||
"""When running in local mode, shutdown the Ray processes.
|
||||
|
||||
This method is used to shutdown processes that were started with
|
||||
services.start_ray_head(). It kills all scheduler, object store, and worker
|
||||
processes that were started by this services module. Driver processes are
|
||||
started and disconnected by worker.py.
|
||||
"""
|
||||
successfully_shut_down = True
|
||||
# Terminate the processes in reverse order.
|
||||
for process_type in all_processes.keys():
|
||||
# Kill all of the processes of a certain type.
|
||||
for p in all_processes[process_type]:
|
||||
success = kill_process(p)
|
||||
successfully_shut_down = successfully_shut_down and success
|
||||
# Reset the list of processes of this type.
|
||||
all_processes[process_type] = []
|
||||
if not successfully_shut_down:
|
||||
logger.warning("Ray did not shut down properly.")
|
||||
|
||||
|
||||
def all_processes_alive(exclude=None):
|
||||
"""Check if all of the processes are still alive.
|
||||
|
||||
Args:
|
||||
exclude: Don't check the processes whose types are in this list.
|
||||
"""
|
||||
|
||||
if exclude is None:
|
||||
exclude = []
|
||||
for process_type, processes in all_processes.items():
|
||||
# Note that p.poll() returns the exit code that the process exited
|
||||
# with, so an exit code of None indicates that the process is still
|
||||
# alive.
|
||||
processes_alive = [p.poll() is None for p in processes]
|
||||
if not all(processes_alive) and process_type not in exclude:
|
||||
logger.warning(
|
||||
"A process of type {} has died.".format(process_type))
|
||||
return False
|
||||
return True
|
||||
if ray.worker._global_node is None:
|
||||
raise Exception("This process is not in a position to determine "
|
||||
"whether all processes are alive or not.")
|
||||
return ray.worker._global_node.remaining_processes_alive()
|
||||
|
||||
|
||||
def address_to_ip(address):
|
||||
@@ -411,7 +325,6 @@ def start_redis(node_ip_address,
|
||||
redis_max_clients=None,
|
||||
redirect_output=False,
|
||||
redirect_worker_output=False,
|
||||
cleanup=True,
|
||||
password=None,
|
||||
use_credis=None,
|
||||
redis_max_memory=None):
|
||||
@@ -434,10 +347,6 @@ def start_redis(node_ip_address,
|
||||
redirect_worker_output (bool): True if worker output should be
|
||||
redirected to a file and false otherwise. Workers will have access
|
||||
to this value when they start up.
|
||||
cleanup (bool): True if using Ray in local mode. If cleanup is true,
|
||||
then all Redis processes started by this method will be killed by
|
||||
services.cleanup() when the Python process that imported services
|
||||
exits.
|
||||
password (str): Prevents external clients without the password
|
||||
from connecting to Redis if provided.
|
||||
use_credis: If True, additionally load the chain-replicated libraries
|
||||
@@ -450,8 +359,9 @@ def start_redis(node_ip_address,
|
||||
capped at 10GB but can be set higher.
|
||||
|
||||
Returns:
|
||||
A tuple of the address for the primary Redis shard and a list of
|
||||
addresses for the remaining shards.
|
||||
A tuple of the address for the primary Redis shard, a list of
|
||||
addresses for the remaining shards, and the processes that were
|
||||
started.
|
||||
"""
|
||||
redis_stdout_file, redis_stderr_file = new_redis_log_file(redirect_output)
|
||||
|
||||
@@ -461,6 +371,8 @@ def start_redis(node_ip_address,
|
||||
raise Exception("The number of Redis shard ports does not match the "
|
||||
"number of Redis shards.")
|
||||
|
||||
processes = []
|
||||
|
||||
if use_credis is None:
|
||||
use_credis = ("RAY_USE_NEW_GCS" in os.environ)
|
||||
if use_credis and password is not None:
|
||||
@@ -471,25 +383,24 @@ def start_redis(node_ip_address,
|
||||
"password-protected Redis ports, ensure that "
|
||||
"the environment variable `RAY_USE_NEW_GCS=off`.")
|
||||
if not use_credis:
|
||||
assigned_port, _ = _start_redis_instance(
|
||||
assigned_port, p = _start_redis_instance(
|
||||
node_ip_address=node_ip_address,
|
||||
port=port,
|
||||
redis_max_clients=redis_max_clients,
|
||||
stdout_file=redis_stdout_file,
|
||||
stderr_file=redis_stderr_file,
|
||||
cleanup=cleanup,
|
||||
password=password,
|
||||
# Below we use None to indicate no limit on the memory of the
|
||||
# primary Redis shard.
|
||||
redis_max_memory=None)
|
||||
processes.append(p)
|
||||
else:
|
||||
assigned_port, _ = _start_redis_instance(
|
||||
assigned_port, p = _start_redis_instance(
|
||||
node_ip_address=node_ip_address,
|
||||
port=port,
|
||||
redis_max_clients=redis_max_clients,
|
||||
stdout_file=redis_stdout_file,
|
||||
stderr_file=redis_stderr_file,
|
||||
cleanup=cleanup,
|
||||
executable=CREDIS_EXECUTABLE,
|
||||
# It is important to load the credis module BEFORE the ray module,
|
||||
# as the latter contains an extern declaration that the former
|
||||
@@ -499,6 +410,7 @@ def start_redis(node_ip_address,
|
||||
# Below we use None to indicate no limit on the memory of the
|
||||
# primary Redis shard.
|
||||
redis_max_memory=None)
|
||||
processes.append(p)
|
||||
if port is not None:
|
||||
assert assigned_port == port
|
||||
port = assigned_port
|
||||
@@ -534,26 +446,25 @@ def start_redis(node_ip_address,
|
||||
redis_stdout_file, redis_stderr_file = new_redis_log_file(
|
||||
redirect_output, shard_number=i)
|
||||
if not use_credis:
|
||||
redis_shard_port, _ = _start_redis_instance(
|
||||
redis_shard_port, p = _start_redis_instance(
|
||||
node_ip_address=node_ip_address,
|
||||
port=redis_shard_ports[i],
|
||||
redis_max_clients=redis_max_clients,
|
||||
stdout_file=redis_stdout_file,
|
||||
stderr_file=redis_stderr_file,
|
||||
cleanup=cleanup,
|
||||
password=password,
|
||||
redis_max_memory=redis_max_memory)
|
||||
processes.append(p)
|
||||
else:
|
||||
assert num_redis_shards == 1, \
|
||||
"For now, RAY_USE_NEW_GCS supports 1 shard, and credis "\
|
||||
"supports 1-node chain for that shard only."
|
||||
redis_shard_port, _ = _start_redis_instance(
|
||||
redis_shard_port, p = _start_redis_instance(
|
||||
node_ip_address=node_ip_address,
|
||||
port=redis_shard_ports[i],
|
||||
redis_max_clients=redis_max_clients,
|
||||
stdout_file=redis_stdout_file,
|
||||
stderr_file=redis_stderr_file,
|
||||
cleanup=cleanup,
|
||||
password=password,
|
||||
executable=CREDIS_EXECUTABLE,
|
||||
# It is important to load the credis module BEFORE the ray
|
||||
@@ -561,6 +472,7 @@ def start_redis(node_ip_address,
|
||||
# former supplies.
|
||||
modules=[CREDIS_MEMBER_MODULE, REDIS_MODULE],
|
||||
redis_max_memory=redis_max_memory)
|
||||
processes.append(p)
|
||||
|
||||
if redis_shard_ports[i] is not None:
|
||||
assert redis_shard_port == redis_shard_ports[i]
|
||||
@@ -578,7 +490,7 @@ def start_redis(node_ip_address,
|
||||
shard_client.execute_command("MEMBER.CONNECT_TO_MASTER",
|
||||
node_ip_address, port)
|
||||
|
||||
return redis_address, redis_shards
|
||||
return redis_address, redis_shards, processes
|
||||
|
||||
|
||||
def _start_redis_instance(node_ip_address="127.0.0.1",
|
||||
@@ -587,7 +499,6 @@ def _start_redis_instance(node_ip_address="127.0.0.1",
|
||||
num_retries=20,
|
||||
stdout_file=None,
|
||||
stderr_file=None,
|
||||
cleanup=True,
|
||||
password=None,
|
||||
executable=REDIS_EXECUTABLE,
|
||||
modules=None,
|
||||
@@ -606,9 +517,6 @@ def _start_redis_instance(node_ip_address="127.0.0.1",
|
||||
no redirection should happen, then this should be None.
|
||||
stderr_file: A file handle opened for writing to redirect stderr to. If
|
||||
no redirection should happen, then this should be None.
|
||||
cleanup (bool): True if using Ray in local mode. If cleanup is true,
|
||||
then this process will be killed by serices.cleanup() when the
|
||||
Python process that imported services exits.
|
||||
password (str): Prevents external clients without the password
|
||||
from connecting to Redis if provided.
|
||||
executable (str): Full path tho the redis-server executable.
|
||||
@@ -659,8 +567,6 @@ def _start_redis_instance(node_ip_address="127.0.0.1",
|
||||
# Check if Redis successfully started (or at least if it the executable
|
||||
# did not exit within 0.1 seconds).
|
||||
if p.poll() is None:
|
||||
if cleanup:
|
||||
all_processes[PROCESS_TYPE_REDIS_SERVER].append(p)
|
||||
break
|
||||
port = new_port()
|
||||
counter += 1
|
||||
@@ -734,7 +640,6 @@ def start_log_monitor(redis_address,
|
||||
node_ip_address,
|
||||
stdout_file=None,
|
||||
stderr_file=None,
|
||||
cleanup=cleanup,
|
||||
redis_password=None):
|
||||
"""Start a log monitor process.
|
||||
|
||||
@@ -746,10 +651,10 @@ def start_log_monitor(redis_address,
|
||||
no redirection should happen, then this should be None.
|
||||
stderr_file: A file handle opened for writing to redirect stderr to. If
|
||||
no redirection should happen, then this should be None.
|
||||
cleanup (bool): True if using Ray in local mode. If cleanup is true,
|
||||
then this process will be killed by services.cleanup() when the
|
||||
Python process that imported services exits.
|
||||
redis_password (str): The password of the redis server.
|
||||
|
||||
Returns:
|
||||
The process that was started.
|
||||
"""
|
||||
log_monitor_filepath = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)), "log_monitor.py")
|
||||
@@ -760,15 +665,14 @@ def start_log_monitor(redis_address,
|
||||
if redis_password:
|
||||
command += ["--redis-password", redis_password]
|
||||
p = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file)
|
||||
if cleanup:
|
||||
all_processes[PROCESS_TYPE_LOG_MONITOR].append(p)
|
||||
record_log_files_in_redis(
|
||||
redis_address,
|
||||
node_ip_address, [stdout_file, stderr_file],
|
||||
password=redis_password)
|
||||
return p
|
||||
|
||||
|
||||
def start_ui(redis_address, stdout_file=None, stderr_file=None, cleanup=True):
|
||||
def start_ui(redis_address, stdout_file=None, stderr_file=None):
|
||||
"""Start a UI process.
|
||||
|
||||
Args:
|
||||
@@ -777,9 +681,9 @@ def start_ui(redis_address, stdout_file=None, stderr_file=None, cleanup=True):
|
||||
no redirection should happen, then this should be None.
|
||||
stderr_file: A file handle opened for writing to redirect stderr to. If
|
||||
no redirection should happen, then this should be None.
|
||||
cleanup (bool): True if using Ray in local mode. If cleanup is true,
|
||||
then this process will be killed by services.cleanup() when the
|
||||
Python process that imported services exits.
|
||||
|
||||
Returns:
|
||||
A tuple of the web UI url and the process that was started.
|
||||
"""
|
||||
|
||||
port = 8888
|
||||
@@ -820,12 +724,11 @@ def start_ui(redis_address, stdout_file=None, stderr_file=None, cleanup=True):
|
||||
logger.warning("Failed to start the UI, you may need to run "
|
||||
"'pip install jupyter'.")
|
||||
else:
|
||||
if cleanup:
|
||||
all_processes[PROCESS_TYPE_WEB_UI].append(ui_process)
|
||||
logger.info("\n" + "=" * 70)
|
||||
logger.info("View the web UI at {}".format(webui_url))
|
||||
logger.info("=" * 70 + "\n")
|
||||
return webui_url
|
||||
return webui_url, ui_process
|
||||
return None, None
|
||||
|
||||
|
||||
def check_and_update_resources(num_cpus, num_gpus, resources):
|
||||
@@ -887,28 +790,40 @@ def check_and_update_resources(num_cpus, num_gpus, resources):
|
||||
return resources
|
||||
|
||||
|
||||
def start_raylet(ray_params,
|
||||
def start_raylet(redis_address,
|
||||
node_ip_address,
|
||||
raylet_name,
|
||||
plasma_store_name,
|
||||
num_initial_workers=0,
|
||||
worker_path,
|
||||
num_cpus=None,
|
||||
num_gpus=None,
|
||||
resources=None,
|
||||
object_manager_port=None,
|
||||
node_manager_port=None,
|
||||
redis_password=None,
|
||||
use_valgrind=False,
|
||||
use_profiler=False,
|
||||
stdout_file=None,
|
||||
stderr_file=None,
|
||||
cleanup=True,
|
||||
config=None):
|
||||
"""Start a raylet, which is a combined local scheduler and object manager.
|
||||
|
||||
Args:
|
||||
ray_params (ray.params.RayParams): The RayParams instance. The
|
||||
following parameters could be checked: redis_address,
|
||||
node_ip_address, worker_path, resources, num_cpus, num_gpus,
|
||||
object_manager_port, node_manager_port, redis_password.
|
||||
resources, object_manager_port, node_manager_port.
|
||||
redis_address (str): The address of the primary Redis server.
|
||||
node_ip_address (str): The IP address of this node.
|
||||
raylet_name (str): The name of the raylet socket to create.
|
||||
plasma_store_name (str): The name of the plasma store socket to connect
|
||||
to.
|
||||
num_initial_workers (int): The number of workers to start initially.
|
||||
worker_path (str): The path of the Python file that new worker
|
||||
processes will execute.
|
||||
num_cpus: The CPUs allocated for this raylet.
|
||||
num_gpus: The GPUs allocated for this raylet.
|
||||
resources: The custom resources allocated for this raylet.
|
||||
object_manager_port: The port to use for the object manager. If this is
|
||||
None, then the object manager will choose its own port.
|
||||
node_manager_port: The port to use for the node manager. If this is
|
||||
None, then the node manager will choose its own port.
|
||||
redis_password: The password to use when connecting to Redis.
|
||||
use_valgrind (bool): True if the raylet should be started inside
|
||||
of valgrind. If this is True, use_profiler must be False.
|
||||
use_profiler (bool): True if the raylet should be started inside
|
||||
@@ -917,14 +832,11 @@ def start_raylet(ray_params,
|
||||
no redirection should happen, then this should be None.
|
||||
stderr_file: A file handle opened for writing to redirect stderr to. If
|
||||
no redirection should happen, then this should be None.
|
||||
cleanup (bool): True if using Ray in local mode. If cleanup is true,
|
||||
then this process will be killed by serices.cleanup() when the
|
||||
Python process that imported services exits.
|
||||
config (dict|None): Optional Raylet configuration that will
|
||||
override defaults in RayConfig.
|
||||
|
||||
Returns:
|
||||
The raylet socket name.
|
||||
The process that was started.
|
||||
"""
|
||||
config = config or {}
|
||||
config_str = ",".join(["{},{}".format(*kv) for kv in config.items()])
|
||||
@@ -932,8 +844,11 @@ def start_raylet(ray_params,
|
||||
if use_valgrind and use_profiler:
|
||||
raise Exception("Cannot use valgrind and profiler at the same time.")
|
||||
|
||||
static_resources = check_and_update_resources(
|
||||
ray_params.num_cpus, ray_params.num_gpus, ray_params.resources)
|
||||
num_initial_workers = (num_cpus if num_cpus is not None else
|
||||
multiprocessing.cpu_count())
|
||||
|
||||
static_resources = check_and_update_resources(num_cpus, num_gpus,
|
||||
resources)
|
||||
|
||||
# Limit the number of workers that can be started in parallel by the
|
||||
# raylet. However, make sure it is at least 1.
|
||||
@@ -944,7 +859,7 @@ def start_raylet(ray_params,
|
||||
resource_argument = ",".join(
|
||||
["{},{}".format(*kv) for kv in static_resources.items()])
|
||||
|
||||
gcs_ip_address, gcs_port = ray_params.redis_address.split(":")
|
||||
gcs_ip_address, gcs_port = redis_address.split(":")
|
||||
|
||||
# Create the command that the Raylet will use to start workers.
|
||||
start_worker_command = ("{} {} "
|
||||
@@ -953,30 +868,28 @@ def start_raylet(ray_params,
|
||||
"--raylet-name={} "
|
||||
"--redis-address={} "
|
||||
"--temp-dir={}".format(
|
||||
sys.executable, ray_params.worker_path,
|
||||
ray_params.node_ip_address, plasma_store_name,
|
||||
raylet_name, ray_params.redis_address,
|
||||
sys.executable, worker_path, node_ip_address,
|
||||
plasma_store_name, raylet_name, redis_address,
|
||||
get_temp_root()))
|
||||
if ray_params.redis_password:
|
||||
start_worker_command += " --redis-password {}".format(
|
||||
ray_params.redis_password)
|
||||
if redis_password:
|
||||
start_worker_command += " --redis-password {}".format(redis_password)
|
||||
|
||||
# If the object manager port is None, then use 0 to cause the object
|
||||
# manager to choose its own port.
|
||||
if ray_params.object_manager_port is None:
|
||||
ray_params.object_manager_port = 0
|
||||
if object_manager_port is None:
|
||||
object_manager_port = 0
|
||||
# If the node manager port is None, then use 0 to cause the node manager
|
||||
# to choose its own port.
|
||||
if ray_params.node_manager_port is None:
|
||||
ray_params.node_manager_port = 0
|
||||
if node_manager_port is None:
|
||||
node_manager_port = 0
|
||||
|
||||
command = [
|
||||
RAYLET_EXECUTABLE,
|
||||
raylet_name,
|
||||
plasma_store_name,
|
||||
str(ray_params.object_manager_port),
|
||||
str(ray_params.node_manager_port),
|
||||
ray_params.node_ip_address,
|
||||
str(object_manager_port),
|
||||
str(node_manager_port),
|
||||
node_ip_address,
|
||||
gcs_ip_address,
|
||||
gcs_port,
|
||||
str(num_initial_workers),
|
||||
@@ -985,12 +898,12 @@ def start_raylet(ray_params,
|
||||
config_str,
|
||||
start_worker_command,
|
||||
"", # Worker command for Java, not needed for Python.
|
||||
ray_params.redis_password or "",
|
||||
redis_password or "",
|
||||
get_temp_root(),
|
||||
]
|
||||
|
||||
if use_valgrind:
|
||||
pid = subprocess.Popen(
|
||||
p = subprocess.Popen(
|
||||
[
|
||||
"valgrind", "--track-origins=yes", "--leak-check=full",
|
||||
"--show-leak-kinds=all", "--leak-check-heuristics=stdstring",
|
||||
@@ -999,7 +912,7 @@ def start_raylet(ray_params,
|
||||
stdout=stdout_file,
|
||||
stderr=stderr_file)
|
||||
elif use_profiler:
|
||||
pid = subprocess.Popen(
|
||||
p = subprocess.Popen(
|
||||
["valgrind", "--tool=callgrind"] + command,
|
||||
stdout=stdout_file,
|
||||
stderr=stderr_file)
|
||||
@@ -1007,19 +920,17 @@ def start_raylet(ray_params,
|
||||
modified_env = os.environ.copy()
|
||||
modified_env["LD_PRELOAD"] = os.environ["RAYLET_PERFTOOLS_PATH"]
|
||||
modified_env["CPUPROFILE"] = os.environ["RAYLET_PERFTOOLS_LOGFILE"]
|
||||
pid = subprocess.Popen(
|
||||
p = subprocess.Popen(
|
||||
command, stdout=stdout_file, stderr=stderr_file, env=modified_env)
|
||||
else:
|
||||
pid = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file)
|
||||
p = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file)
|
||||
|
||||
if cleanup:
|
||||
all_processes[PROCESS_TYPE_RAYLET].append(pid)
|
||||
record_log_files_in_redis(
|
||||
ray_params.redis_address,
|
||||
ray_params.node_ip_address, [stdout_file, stderr_file],
|
||||
password=ray_params.redis_password)
|
||||
redis_address,
|
||||
node_ip_address, [stdout_file, stderr_file],
|
||||
password=redis_password)
|
||||
|
||||
return raylet_name
|
||||
return p
|
||||
|
||||
|
||||
def determine_plasma_store_config(object_store_memory=None,
|
||||
@@ -1104,11 +1015,9 @@ def determine_plasma_store_config(object_store_memory=None,
|
||||
|
||||
def start_plasma_store(node_ip_address,
|
||||
redis_address,
|
||||
object_manager_port=None,
|
||||
store_stdout_file=None,
|
||||
store_stderr_file=None,
|
||||
stdout_file=None,
|
||||
stderr_file=None,
|
||||
object_store_memory=None,
|
||||
cleanup=True,
|
||||
plasma_directory=None,
|
||||
huge_pages=False,
|
||||
plasma_store_socket_name=None,
|
||||
@@ -1119,25 +1028,20 @@ def start_plasma_store(node_ip_address,
|
||||
node_ip_address (str): The IP address of the node running the object
|
||||
store.
|
||||
redis_address (str): The address of the Redis instance to connect to.
|
||||
object_manager_port (int): The port to use for the object manager. If
|
||||
this is not provided, one will be generated randomly.
|
||||
store_stdout_file: A file handle opened for writing to redirect stdout
|
||||
stdout_file: A file handle opened for writing to redirect stdout
|
||||
to. If no redirection should happen, then this should be None.
|
||||
store_stderr_file: A file handle opened for writing to redirect stderr
|
||||
stderr_file: A file handle opened for writing to redirect stderr
|
||||
to. If no redirection should happen, then this should be None.
|
||||
object_store_memory: The amount of memory (in bytes) to start the
|
||||
object store with.
|
||||
cleanup (bool): True if using Ray in local mode. If cleanup is true,
|
||||
then this process will be killed by serices.cleanup() when the
|
||||
Python process that imported services exits.
|
||||
plasma_directory: A directory where the Plasma memory mapped files will
|
||||
be created.
|
||||
huge_pages: Boolean flag indicating whether to start the Object
|
||||
Store with hugetlbfs support. Requires plasma_directory.
|
||||
redis_password (str): The password of the redis server.
|
||||
|
||||
Return:
|
||||
The Plasma store socket name.
|
||||
Returns:
|
||||
The process that was started.
|
||||
"""
|
||||
object_store_memory, plasma_directory = determine_plasma_store_config(
|
||||
object_store_memory, plasma_directory, huge_pages)
|
||||
@@ -1153,23 +1057,21 @@ def start_plasma_store(node_ip_address,
|
||||
logger.info("Starting the Plasma object store with {} GB memory "
|
||||
"using {}.".format(object_store_memory_str, plasma_directory))
|
||||
# Start the Plasma store.
|
||||
plasma_store_name, p1 = ray.plasma.start_plasma_store(
|
||||
plasma_store_name, p = ray.plasma.start_plasma_store(
|
||||
plasma_store_memory=object_store_memory,
|
||||
use_profiler=RUN_PLASMA_STORE_PROFILER,
|
||||
stdout_file=store_stdout_file,
|
||||
stderr_file=store_stderr_file,
|
||||
stdout_file=stdout_file,
|
||||
stderr_file=stderr_file,
|
||||
plasma_directory=plasma_directory,
|
||||
huge_pages=huge_pages,
|
||||
socket_name=plasma_store_socket_name)
|
||||
|
||||
if cleanup:
|
||||
all_processes[PROCESS_TYPE_PLASMA_STORE].append(p1)
|
||||
record_log_files_in_redis(
|
||||
redis_address,
|
||||
node_ip_address, [store_stdout_file, store_stderr_file],
|
||||
node_ip_address, [stdout_file, stderr_file],
|
||||
password=redis_password)
|
||||
|
||||
return plasma_store_name
|
||||
return p
|
||||
|
||||
|
||||
def start_worker(node_ip_address,
|
||||
@@ -1178,8 +1080,7 @@ def start_worker(node_ip_address,
|
||||
redis_address,
|
||||
worker_path,
|
||||
stdout_file=None,
|
||||
stderr_file=None,
|
||||
cleanup=True):
|
||||
stderr_file=None):
|
||||
"""This method starts a worker process.
|
||||
|
||||
Args:
|
||||
@@ -1194,10 +1095,9 @@ def start_worker(node_ip_address,
|
||||
no redirection should happen, then this should be None.
|
||||
stderr_file: A file handle opened for writing to redirect stderr to. If
|
||||
no redirection should happen, then this should be None.
|
||||
cleanup (bool): True if using Ray in local mode. If cleanup is true,
|
||||
then this process will be killed by services.cleanup() when the
|
||||
Python process that imported services exits. This is True by
|
||||
default.
|
||||
|
||||
Returns:
|
||||
The process that was started.
|
||||
"""
|
||||
command = [
|
||||
sys.executable, "-u", worker_path,
|
||||
@@ -1207,17 +1107,15 @@ def start_worker(node_ip_address,
|
||||
"--temp-dir=" + get_temp_root()
|
||||
]
|
||||
p = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file)
|
||||
if cleanup:
|
||||
all_processes[PROCESS_TYPE_WORKER].append(p)
|
||||
record_log_files_in_redis(redis_address, node_ip_address,
|
||||
[stdout_file, stderr_file])
|
||||
return p
|
||||
|
||||
|
||||
def start_monitor(redis_address,
|
||||
node_ip_address,
|
||||
stdout_file=None,
|
||||
stderr_file=None,
|
||||
cleanup=True,
|
||||
autoscaling_config=None,
|
||||
redis_password=None):
|
||||
"""Run a process to monitor the other processes.
|
||||
@@ -1230,12 +1128,11 @@ def start_monitor(redis_address,
|
||||
no redirection should happen, then this should be None.
|
||||
stderr_file: A file handle opened for writing to redirect stderr to. If
|
||||
no redirection should happen, then this should be None.
|
||||
cleanup (bool): True if using Ray in local mode. If cleanup is true,
|
||||
then this process will be killed by services.cleanup() when the
|
||||
Python process that imported services exits. This is True by
|
||||
default.
|
||||
autoscaling_config: path to autoscaling config file.
|
||||
redis_password (str): The password of the redis server.
|
||||
|
||||
Returns:
|
||||
The process that was started.
|
||||
"""
|
||||
monitor_path = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)), "monitor.py")
|
||||
@@ -1248,18 +1145,16 @@ def start_monitor(redis_address,
|
||||
if redis_password:
|
||||
command.append("--redis-password=" + redis_password)
|
||||
p = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file)
|
||||
if cleanup:
|
||||
all_processes[PROCESS_TYPE_MONITOR].append(p)
|
||||
record_log_files_in_redis(
|
||||
redis_address,
|
||||
node_ip_address, [stdout_file, stderr_file],
|
||||
password=redis_password)
|
||||
return p
|
||||
|
||||
|
||||
def start_raylet_monitor(redis_address,
|
||||
stdout_file=None,
|
||||
stderr_file=None,
|
||||
cleanup=True,
|
||||
redis_password=None,
|
||||
config=None):
|
||||
"""Run a process to monitor the other processes.
|
||||
@@ -1270,13 +1165,12 @@ def start_raylet_monitor(redis_address,
|
||||
no redirection should happen, then this should be None.
|
||||
stderr_file: A file handle opened for writing to redirect stderr to. If
|
||||
no redirection should happen, then this should be None.
|
||||
cleanup (bool): True if using Ray in local mode. If cleanup is true,
|
||||
then this process will be killed by services.cleanup() when the
|
||||
Python process that imported services exits. This is True by
|
||||
default.
|
||||
redis_password (str): The password of the redis server.
|
||||
config (dict|None): Optional configuration that will
|
||||
override defaults in RayConfig.
|
||||
|
||||
Returns:
|
||||
The process that was started.
|
||||
"""
|
||||
gcs_ip_address, gcs_port = redis_address.split(":")
|
||||
redis_password = redis_password or ""
|
||||
@@ -1286,222 +1180,4 @@ def start_raylet_monitor(redis_address,
|
||||
if redis_password:
|
||||
command += [redis_password]
|
||||
p = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file)
|
||||
if cleanup:
|
||||
all_processes[PROCESS_TYPE_MONITOR].append(p)
|
||||
|
||||
|
||||
def start_ray_processes(ray_params, cleanup=True):
|
||||
"""Helper method to start Ray processes.
|
||||
|
||||
Args:
|
||||
ray_params (ray.params.RayParams): The RayParams instance. The
|
||||
following parameters will be set to default values if it's None:
|
||||
node_ip_address("127.0.0.1"), include_webui(False),
|
||||
worker_path(path of default_worker.py), include_log_monitor(False)
|
||||
cleanup (bool): If cleanup is true, then the processes started here
|
||||
will be killed by services.cleanup() when the Python process that
|
||||
called this method exits.
|
||||
|
||||
Returns:
|
||||
A dictionary of the address information for the processes that were
|
||||
started.
|
||||
"""
|
||||
|
||||
set_temp_root(ray_params.temp_dir)
|
||||
|
||||
logger.info("Process STDOUT and STDERR is being redirected to {}.".format(
|
||||
get_logs_dir_path()))
|
||||
|
||||
config = json.loads(
|
||||
ray_params._internal_config) if ray_params._internal_config else None
|
||||
|
||||
ray_params.update_if_absent(
|
||||
include_log_monitor=False,
|
||||
resources={},
|
||||
include_webui=False,
|
||||
node_ip_address="127.0.0.1")
|
||||
|
||||
if ray_params.num_workers is not None:
|
||||
raise Exception("The 'num_workers' argument is deprecated. Please use "
|
||||
"'num_cpus' instead.")
|
||||
else:
|
||||
num_initial_workers = (ray_params.num_cpus
|
||||
if ray_params.num_cpus is not None else
|
||||
multiprocessing.cpu_count())
|
||||
|
||||
ray_params.update_if_absent(
|
||||
address_info={},
|
||||
worker_path=os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)),
|
||||
"workers/default_worker.py"))
|
||||
ray_params.address_info["node_ip_address"] = ray_params.node_ip_address
|
||||
|
||||
# Start Redis if there isn't already an instance running. TODO(rkn): We are
|
||||
# suppressing the output of Redis because on Linux it prints a bunch of
|
||||
# warning messages when it starts up. Instead of suppressing the output, we
|
||||
# should address the warnings.
|
||||
ray_params.redis_address = ray_params.address_info.get("redis_address")
|
||||
ray_params.redis_shards = ray_params.address_info.get("redis_shards", [])
|
||||
if ray_params.redis_address is None:
|
||||
ray_params.redis_address, ray_params.redis_shards = start_redis(
|
||||
ray_params.node_ip_address,
|
||||
port=ray_params.redis_port,
|
||||
redis_shard_ports=ray_params.redis_shard_ports,
|
||||
num_redis_shards=ray_params.num_redis_shards,
|
||||
redis_max_clients=ray_params.redis_max_clients,
|
||||
redirect_output=True,
|
||||
redirect_worker_output=ray_params.redirect_worker_output,
|
||||
cleanup=cleanup,
|
||||
password=ray_params.redis_password,
|
||||
redis_max_memory=ray_params.redis_max_memory)
|
||||
ray_params.address_info["redis_address"] = ray_params.redis_address
|
||||
time.sleep(0.1)
|
||||
|
||||
# Start monitoring the processes.
|
||||
monitor_stdout_file, monitor_stderr_file = new_monitor_log_file(
|
||||
ray_params.redirect_output)
|
||||
start_monitor(
|
||||
ray_params.redis_address,
|
||||
ray_params.node_ip_address,
|
||||
stdout_file=monitor_stdout_file,
|
||||
stderr_file=monitor_stderr_file,
|
||||
cleanup=cleanup,
|
||||
autoscaling_config=ray_params.autoscaling_config,
|
||||
redis_password=ray_params.redis_password)
|
||||
start_raylet_monitor(
|
||||
ray_params.redis_address,
|
||||
stdout_file=monitor_stdout_file,
|
||||
stderr_file=monitor_stderr_file,
|
||||
cleanup=cleanup,
|
||||
redis_password=ray_params.redis_password,
|
||||
config=config)
|
||||
if ray_params.redis_shards == []:
|
||||
# Get redis shards from primary redis instance.
|
||||
redis_ip_address, redis_port = ray_params.redis_address.split(":")
|
||||
redis_client = redis.StrictRedis(
|
||||
host=redis_ip_address,
|
||||
port=redis_port,
|
||||
password=ray_params.redis_password)
|
||||
redis_shards = redis_client.lrange("RedisShards", start=0, end=-1)
|
||||
ray_params.redis_shards = [
|
||||
ray.utils.decode(shard) for shard in redis_shards
|
||||
]
|
||||
ray_params.address_info["redis_shards"] = ray_params.redis_shards
|
||||
|
||||
# Start the log monitor, if necessary.
|
||||
if ray_params.include_log_monitor:
|
||||
log_monitor_stdout_file, log_monitor_stderr_file = (
|
||||
new_log_monitor_log_file())
|
||||
start_log_monitor(
|
||||
ray_params.redis_address,
|
||||
ray_params.node_ip_address,
|
||||
stdout_file=log_monitor_stdout_file,
|
||||
stderr_file=log_monitor_stderr_file,
|
||||
cleanup=cleanup,
|
||||
redis_password=ray_params.redis_password)
|
||||
|
||||
# Initialize with existing services.
|
||||
object_store_address = ray_params.address_info.get("object_store_address")
|
||||
raylet_socket_name = ray_params.address_info.get("raylet_socket_name")
|
||||
|
||||
# Start the object store.
|
||||
assert object_store_address is None
|
||||
# Start Plasma.
|
||||
plasma_store_stdout_file, plasma_store_stderr_file = (
|
||||
new_plasma_store_log_file(ray_params.redirect_output))
|
||||
|
||||
ray_params.address_info["object_store_address"] = start_plasma_store(
|
||||
ray_params.node_ip_address,
|
||||
ray_params.redis_address,
|
||||
store_stdout_file=plasma_store_stdout_file,
|
||||
store_stderr_file=plasma_store_stderr_file,
|
||||
object_store_memory=ray_params.object_store_memory,
|
||||
cleanup=cleanup,
|
||||
plasma_directory=ray_params.plasma_directory,
|
||||
huge_pages=ray_params.huge_pages,
|
||||
plasma_store_socket_name=ray_params.plasma_store_socket_name,
|
||||
redis_password=ray_params.redis_password)
|
||||
time.sleep(0.1)
|
||||
|
||||
# Start the raylet.
|
||||
assert raylet_socket_name is None
|
||||
raylet_stdout_file, raylet_stderr_file = new_raylet_log_file(
|
||||
redirect_output=ray_params.redirect_worker_output)
|
||||
ray_params.address_info["raylet_socket_name"] = start_raylet(
|
||||
ray_params,
|
||||
ray_params.raylet_socket_name or get_raylet_socket_name(),
|
||||
ray_params.address_info["object_store_address"],
|
||||
num_initial_workers=num_initial_workers,
|
||||
stdout_file=raylet_stdout_file,
|
||||
stderr_file=raylet_stderr_file,
|
||||
cleanup=cleanup,
|
||||
config=config)
|
||||
|
||||
# Try to start the web UI.
|
||||
if ray_params.include_webui:
|
||||
ui_stdout_file, ui_stderr_file = new_webui_log_file()
|
||||
ray_params.address_info["webui_url"] = start_ui(
|
||||
ray_params.redis_address,
|
||||
stdout_file=ui_stdout_file,
|
||||
stderr_file=ui_stderr_file,
|
||||
cleanup=cleanup)
|
||||
else:
|
||||
ray_params.address_info["webui_url"] = ""
|
||||
# Return the addresses of the relevant processes.
|
||||
return ray_params.address_info
|
||||
|
||||
|
||||
def start_ray_node(ray_params, cleanup=True):
|
||||
"""Start the Ray processes for a single node.
|
||||
|
||||
This assumes that the Ray processes on some master node have already been
|
||||
started.
|
||||
|
||||
Args:
|
||||
ray_params (ray.params.RayParams): The RayParams instance. The
|
||||
following parameters could be checked: node_ip_address,
|
||||
redis_address, object_manager_port, node_manager_port,
|
||||
num_workers, object_store_memory, redis_password, worker_path,
|
||||
cleanup, redirect_worker_output, redirect_output, resources,
|
||||
plasma_directory, huge_pages, plasma_store_socket_name,
|
||||
raylet_socket_name, temp_dir, _internal_config.
|
||||
cleanup (bool): If cleanup is true, then the processes started here
|
||||
will be killed by services.cleanup() when the Python process that
|
||||
called this method exits.
|
||||
|
||||
Returns:
|
||||
A dictionary of the address information for the processes that were
|
||||
started.
|
||||
"""
|
||||
ray_params.address_info = {
|
||||
"redis_address": ray_params.redis_address,
|
||||
}
|
||||
ray_params.update(include_log_monitor=True)
|
||||
return start_ray_processes(ray_params, cleanup=cleanup)
|
||||
|
||||
|
||||
def start_ray_head(ray_params, cleanup=True):
|
||||
"""Start Ray in local mode.
|
||||
|
||||
Args:
|
||||
ray_params (ray.params.RayParams): The RayParams instance. The
|
||||
following parameters could be checked: address_info,
|
||||
object_manager_port, node_manager_port, node_ip_address,
|
||||
redis_port, redis_shard_ports, num_workers, object_store_memory,
|
||||
redis_max_memory, worker_path, cleanup, redirect_worker_output,
|
||||
redirect_output, start_workers_from_local_scheduler, resources,
|
||||
num_redis_shards, redis_max_clients, redis_password, include_webui,
|
||||
huge_pages, plasma_directory, autoscaling_config,
|
||||
plasma_store_socket_name, raylet_socket_name, temp_dir,
|
||||
_internal_config.
|
||||
cleanup (bool): If cleanup is true, then the processes started here
|
||||
will be killed by services.cleanup() when the Python process that
|
||||
called this method exits.
|
||||
|
||||
Returns:
|
||||
A dictionary of the address information for the processes that were
|
||||
started.
|
||||
"""
|
||||
ray_params.update_if_absent(num_redis_shards=1, include_webui=True)
|
||||
ray_params.update(include_log_monitor=True)
|
||||
return start_ray_processes(ray_params, cleanup=cleanup)
|
||||
return p
|
||||
|
||||
Reference in New Issue
Block a user