mirror of
https://github.com/wassname/ray.git
synced 2026-07-13 05:16:56 +08:00
Lint Python files with Yapf (#1872)
This commit is contained in:
Philipp Moritz
committed by
Robert Nishihara
Robert Nishihara
parent
a3ddde398c
commit
74162d1492
+237
-200
@@ -41,16 +41,12 @@ PROCESS_TYPE_WEB_UI = "web_ui"
|
||||
# important because it determines the order in which these processes will be
|
||||
# terminated when Ray exits, and certain orders will cause errors to be logged
|
||||
# to the screen.
|
||||
all_processes = OrderedDict([(PROCESS_TYPE_MONITOR, []),
|
||||
(PROCESS_TYPE_LOG_MONITOR, []),
|
||||
(PROCESS_TYPE_WORKER, []),
|
||||
(PROCESS_TYPE_RAYLET, []),
|
||||
(PROCESS_TYPE_LOCAL_SCHEDULER, []),
|
||||
(PROCESS_TYPE_PLASMA_MANAGER, []),
|
||||
(PROCESS_TYPE_PLASMA_STORE, []),
|
||||
(PROCESS_TYPE_GLOBAL_SCHEDULER, []),
|
||||
(PROCESS_TYPE_REDIS_SERVER, []),
|
||||
(PROCESS_TYPE_WEB_UI, [])],)
|
||||
all_processes = OrderedDict(
|
||||
[(PROCESS_TYPE_MONITOR, []), (PROCESS_TYPE_LOG_MONITOR, []),
|
||||
(PROCESS_TYPE_WORKER, []), (PROCESS_TYPE_RAYLET, []),
|
||||
(PROCESS_TYPE_LOCAL_SCHEDULER, []), (PROCESS_TYPE_PLASMA_MANAGER, []),
|
||||
(PROCESS_TYPE_PLASMA_STORE, []), (PROCESS_TYPE_GLOBAL_SCHEDULER, []),
|
||||
(PROCESS_TYPE_REDIS_SERVER, []), (PROCESS_TYPE_WEB_UI, [])], )
|
||||
|
||||
# True if processes are run in the valgrind profiler.
|
||||
RUN_RAYLET_PROFILER = False
|
||||
@@ -82,17 +78,15 @@ RAYLET_MONITOR_EXECUTABLE = os.path.join(
|
||||
os.path.abspath(os.path.dirname(__file__)),
|
||||
"core/src/ray/raylet/raylet_monitor")
|
||||
RAYLET_EXECUTABLE = os.path.join(
|
||||
os.path.abspath(os.path.dirname(__file__)),
|
||||
"core/src/ray/raylet/raylet")
|
||||
os.path.abspath(os.path.dirname(__file__)), "core/src/ray/raylet/raylet")
|
||||
|
||||
# ObjectStoreAddress tuples contain all information necessary to connect to an
|
||||
# object store. The fields are:
|
||||
# - name: The socket name for the object store
|
||||
# - manager_name: The socket name for the object store manager
|
||||
# - manager_port: The Internet port that the object store manager listens on
|
||||
ObjectStoreAddress = namedtuple("ObjectStoreAddress", ["name",
|
||||
"manager_name",
|
||||
"manager_port"])
|
||||
ObjectStoreAddress = namedtuple("ObjectStoreAddress",
|
||||
["name", "manager_name", "manager_port"])
|
||||
|
||||
|
||||
def address(ip_address, port):
|
||||
@@ -133,8 +127,10 @@ def kill_process(p):
|
||||
if p.poll() is not None:
|
||||
# The process has already terminated.
|
||||
return True
|
||||
if any([RUN_RAYLET_PROFILER, RUN_LOCAL_SCHEDULER_PROFILER,
|
||||
RUN_PLASMA_MANAGER_PROFILER, RUN_PLASMA_STORE_PROFILER]):
|
||||
if any([
|
||||
RUN_RAYLET_PROFILER, RUN_LOCAL_SCHEDULER_PROFILER,
|
||||
RUN_PLASMA_MANAGER_PROFILER, RUN_PLASMA_STORE_PROFILER
|
||||
]):
|
||||
# Give process signal to write profiler data.
|
||||
os.kill(p.pid, signal.SIGINT)
|
||||
# Wait for profiling data to be written.
|
||||
@@ -260,8 +256,8 @@ def record_log_files_in_redis(redis_address, node_ip_address, log_files):
|
||||
for log_file in log_files:
|
||||
if log_file is not None:
|
||||
redis_ip_address, redis_port = redis_address.split(":")
|
||||
redis_client = redis.StrictRedis(host=redis_ip_address,
|
||||
port=redis_port)
|
||||
redis_client = redis.StrictRedis(
|
||||
host=redis_ip_address, port=redis_port)
|
||||
# The name of the key storing the list of log filenames for this IP
|
||||
# address.
|
||||
log_file_list_key = "LOG_FILENAMES:{}".format(node_ip_address)
|
||||
@@ -304,8 +300,8 @@ def wait_for_redis_to_start(redis_ip_address, redis_port, num_retries=5):
|
||||
while counter < num_retries:
|
||||
try:
|
||||
# Run some random command and see if it worked.
|
||||
print("Waiting for redis server at {}:{} to respond..."
|
||||
.format(redis_ip_address, redis_port))
|
||||
print("Waiting for redis server at {}:{} to respond...".format(
|
||||
redis_ip_address, redis_port))
|
||||
redis_client.client_list()
|
||||
except redis.ConnectionError as e:
|
||||
# Wait a little bit.
|
||||
@@ -427,17 +423,19 @@ def start_credis(node_ip_address,
|
||||
"""
|
||||
|
||||
components = ["credis_master", "credis_head", "credis_tail"]
|
||||
modules = [CREDIS_MASTER_MODULE, CREDIS_MEMBER_MODULE,
|
||||
CREDIS_MEMBER_MODULE]
|
||||
modules = [
|
||||
CREDIS_MASTER_MODULE, CREDIS_MEMBER_MODULE, CREDIS_MEMBER_MODULE
|
||||
]
|
||||
ports = []
|
||||
|
||||
for i, component in enumerate(components):
|
||||
stdout_file, stderr_file = new_log_files(
|
||||
component, redirect_output)
|
||||
stdout_file, stderr_file = new_log_files(component, redirect_output)
|
||||
|
||||
new_port, _ = start_redis_instance(
|
||||
node_ip_address=node_ip_address, port=port,
|
||||
stdout_file=stdout_file, stderr_file=stderr_file,
|
||||
node_ip_address=node_ip_address,
|
||||
port=port,
|
||||
stdout_file=stdout_file,
|
||||
stderr_file=stderr_file,
|
||||
cleanup=cleanup,
|
||||
module=modules[i],
|
||||
executable=CREDIS_EXECUTABLE)
|
||||
@@ -456,8 +454,7 @@ def start_credis(node_ip_address,
|
||||
|
||||
# Register credis master in redis
|
||||
redis_ip_address, redis_port = redis_address.split(":")
|
||||
redis_client = redis.StrictRedis(host=redis_ip_address,
|
||||
port=redis_port)
|
||||
redis_client = redis.StrictRedis(host=redis_ip_address, port=redis_port)
|
||||
redis_client.set("credis_address", credis_address)
|
||||
|
||||
return credis_address
|
||||
@@ -509,9 +506,11 @@ def start_redis(node_ip_address,
|
||||
"number of Redis shards.")
|
||||
|
||||
assigned_port, _ = start_redis_instance(
|
||||
node_ip_address=node_ip_address, port=port,
|
||||
node_ip_address=node_ip_address,
|
||||
port=port,
|
||||
redis_max_clients=redis_max_clients,
|
||||
stdout_file=redis_stdout_file, stderr_file=redis_stderr_file,
|
||||
stdout_file=redis_stdout_file,
|
||||
stderr_file=redis_stderr_file,
|
||||
cleanup=cleanup)
|
||||
if port is not None:
|
||||
assert assigned_port == port
|
||||
@@ -540,7 +539,8 @@ def start_redis(node_ip_address,
|
||||
node_ip_address=node_ip_address,
|
||||
port=redis_shard_ports[i],
|
||||
redis_max_clients=redis_max_clients,
|
||||
stdout_file=redis_stdout_file, stderr_file=redis_stderr_file,
|
||||
stdout_file=redis_stdout_file,
|
||||
stderr_file=redis_stderr_file,
|
||||
cleanup=cleanup)
|
||||
if redis_shard_ports[i] is not None:
|
||||
assert redis_shard_port == redis_shard_ports[i]
|
||||
@@ -601,11 +601,13 @@ def start_redis_instance(node_ip_address="127.0.0.1",
|
||||
while counter < num_retries:
|
||||
if counter > 0:
|
||||
print("Redis failed to start, retrying now.")
|
||||
p = subprocess.Popen([executable,
|
||||
"--port", str(port),
|
||||
"--loglevel", "warning",
|
||||
"--loadmodule", module],
|
||||
stdout=stdout_file, stderr=stderr_file)
|
||||
p = subprocess.Popen(
|
||||
[
|
||||
executable, "--port",
|
||||
str(port), "--loglevel", "warning", "--loadmodule", module
|
||||
],
|
||||
stdout=stdout_file,
|
||||
stderr=stderr_file)
|
||||
time.sleep(0.1)
|
||||
# Check if Redis successfully started (or at least if it the executable
|
||||
# did not exit within 0.1 seconds).
|
||||
@@ -652,8 +654,8 @@ def start_redis_instance(node_ip_address="127.0.0.1",
|
||||
# Increase the hard and soft limits for the redis client pubsub buffer to
|
||||
# 128MB. This is a hack to make it less likely for pubsub messages to be
|
||||
# dropped and for pubsub connections to therefore be killed.
|
||||
cur_config = (redis_client.config_get("client-output-buffer-limit")
|
||||
["client-output-buffer-limit"])
|
||||
cur_config = (redis_client.config_get("client-output-buffer-limit")[
|
||||
"client-output-buffer-limit"])
|
||||
cur_config_list = cur_config.split()
|
||||
assert len(cur_config_list) == 12
|
||||
cur_config_list[8:] = ["pubsub", "134217728", "134217728", "60"]
|
||||
@@ -662,13 +664,17 @@ def start_redis_instance(node_ip_address="127.0.0.1",
|
||||
# Put a time stamp in Redis to indicate when it was started.
|
||||
redis_client.set("redis_start_time", time.time())
|
||||
# Record the log files in Redis.
|
||||
record_log_files_in_redis(address(node_ip_address, port), node_ip_address,
|
||||
[stdout_file, stderr_file])
|
||||
record_log_files_in_redis(
|
||||
address(node_ip_address, port), node_ip_address,
|
||||
[stdout_file, stderr_file])
|
||||
return port, p
|
||||
|
||||
|
||||
def start_log_monitor(redis_address, node_ip_address, stdout_file=None,
|
||||
stderr_file=None, cleanup=cleanup):
|
||||
def start_log_monitor(redis_address,
|
||||
node_ip_address,
|
||||
stdout_file=None,
|
||||
stderr_file=None,
|
||||
cleanup=cleanup):
|
||||
"""Start a log monitor process.
|
||||
|
||||
Args:
|
||||
@@ -684,20 +690,25 @@ def start_log_monitor(redis_address, node_ip_address, stdout_file=None,
|
||||
Python process that imported services exits.
|
||||
"""
|
||||
log_monitor_filepath = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)),
|
||||
"log_monitor.py")
|
||||
p = subprocess.Popen([sys.executable, "-u", log_monitor_filepath,
|
||||
"--redis-address", redis_address,
|
||||
"--node-ip-address", node_ip_address],
|
||||
stdout=stdout_file, stderr=stderr_file)
|
||||
os.path.dirname(os.path.abspath(__file__)), "log_monitor.py")
|
||||
p = subprocess.Popen(
|
||||
[
|
||||
sys.executable, "-u", log_monitor_filepath, "--redis-address",
|
||||
redis_address, "--node-ip-address", node_ip_address
|
||||
],
|
||||
stdout=stdout_file,
|
||||
stderr=stderr_file)
|
||||
if cleanup:
|
||||
all_processes[PROCESS_TYPE_LOG_MONITOR].append(p)
|
||||
record_log_files_in_redis(redis_address, node_ip_address,
|
||||
[stdout_file, stderr_file])
|
||||
|
||||
|
||||
def start_global_scheduler(redis_address, node_ip_address,
|
||||
stdout_file=None, stderr_file=None, cleanup=True):
|
||||
def start_global_scheduler(redis_address,
|
||||
node_ip_address,
|
||||
stdout_file=None,
|
||||
stderr_file=None,
|
||||
cleanup=True):
|
||||
"""Start a global scheduler process.
|
||||
|
||||
Args:
|
||||
@@ -712,10 +723,11 @@ def start_global_scheduler(redis_address, node_ip_address,
|
||||
then this process will be killed by services.cleanup() when the
|
||||
Python process that imported services exits.
|
||||
"""
|
||||
p = global_scheduler.start_global_scheduler(redis_address,
|
||||
node_ip_address,
|
||||
stdout_file=stdout_file,
|
||||
stderr_file=stderr_file)
|
||||
p = global_scheduler.start_global_scheduler(
|
||||
redis_address,
|
||||
node_ip_address,
|
||||
stdout_file=stdout_file,
|
||||
stderr_file=stderr_file)
|
||||
if cleanup:
|
||||
all_processes[PROCESS_TYPE_GLOBAL_SCHEDULER].append(p)
|
||||
record_log_files_in_redis(redis_address, node_ip_address,
|
||||
@@ -737,8 +749,7 @@ def start_ui(redis_address, stdout_file=None, stderr_file=None, cleanup=True):
|
||||
"""
|
||||
new_env = os.environ.copy()
|
||||
notebook_filepath = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)),
|
||||
"WebUI.ipynb")
|
||||
os.path.dirname(os.path.abspath(__file__)), "WebUI.ipynb")
|
||||
# We copy the notebook file so that the original doesn't get modified by
|
||||
# the user.
|
||||
random_ui_id = random.randint(0, 100000)
|
||||
@@ -759,19 +770,23 @@ def start_ui(redis_address, stdout_file=None, stderr_file=None, cleanup=True):
|
||||
# We generate the token used for authentication ourselves to avoid
|
||||
# querying the jupyter server.
|
||||
token = binascii.hexlify(os.urandom(24)).decode("ascii")
|
||||
command = ["jupyter", "notebook", "--no-browser",
|
||||
"--port={}".format(port),
|
||||
"--NotebookApp.iopub_data_rate_limit=10000000000",
|
||||
"--NotebookApp.open_browser=False",
|
||||
"--NotebookApp.token={}".format(token)]
|
||||
command = [
|
||||
"jupyter", "notebook", "--no-browser", "--port={}".format(port),
|
||||
"--NotebookApp.iopub_data_rate_limit=10000000000",
|
||||
"--NotebookApp.open_browser=False",
|
||||
"--NotebookApp.token={}".format(token)
|
||||
]
|
||||
# If the user is root, add the --allow-root flag.
|
||||
if os.geteuid() == 0:
|
||||
command.append("--allow-root")
|
||||
|
||||
try:
|
||||
ui_process = subprocess.Popen(command, env=new_env,
|
||||
cwd=new_notebook_directory,
|
||||
stdout=stdout_file, stderr=stderr_file)
|
||||
ui_process = subprocess.Popen(
|
||||
command,
|
||||
env=new_env,
|
||||
cwd=new_notebook_directory,
|
||||
stdout=stdout_file,
|
||||
stderr=stderr_file)
|
||||
except Exception:
|
||||
print("Failed to start the UI, you may need to run "
|
||||
"'pip install jupyter'.")
|
||||
@@ -836,8 +851,8 @@ def start_local_scheduler(redis_address,
|
||||
|
||||
# Check that the number of GPUs that the local scheduler wants doesn't
|
||||
# excede the amount allowed by CUDA_VISIBLE_DEVICES.
|
||||
if ("GPU" in resources and gpu_ids is not None and
|
||||
resources["GPU"] > len(gpu_ids)):
|
||||
if ("GPU" in resources and gpu_ids is not None
|
||||
and resources["GPU"] > len(gpu_ids)):
|
||||
raise Exception("Attempting to start local scheduler with {} GPUs, "
|
||||
"but CUDA_VISIBLE_DEVICES contains {}.".format(
|
||||
resources["GPU"], gpu_ids))
|
||||
@@ -906,21 +921,14 @@ def start_raylet(redis_address,
|
||||
"--node-ip-address={} "
|
||||
"--object-store-name={} "
|
||||
"--raylet-name={} "
|
||||
"--redis-address={}"
|
||||
.format(sys.executable,
|
||||
worker_path,
|
||||
node_ip_address,
|
||||
plasma_store_name,
|
||||
raylet_name,
|
||||
redis_address))
|
||||
"--redis-address={}".format(
|
||||
sys.executable, worker_path, node_ip_address,
|
||||
plasma_store_name, raylet_name, redis_address))
|
||||
|
||||
command = [RAYLET_EXECUTABLE,
|
||||
raylet_name,
|
||||
plasma_store_name,
|
||||
node_ip_address,
|
||||
gcs_ip_address,
|
||||
gcs_port,
|
||||
start_worker_command]
|
||||
command = [
|
||||
RAYLET_EXECUTABLE, raylet_name, plasma_store_name, node_ip_address,
|
||||
gcs_ip_address, gcs_port, start_worker_command
|
||||
]
|
||||
pid = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file)
|
||||
|
||||
if cleanup:
|
||||
@@ -931,12 +939,18 @@ def start_raylet(redis_address,
|
||||
return raylet_name
|
||||
|
||||
|
||||
def start_objstore(node_ip_address, redis_address,
|
||||
object_manager_port=None, store_stdout_file=None,
|
||||
store_stderr_file=None, manager_stdout_file=None,
|
||||
manager_stderr_file=None, objstore_memory=None,
|
||||
cleanup=True, plasma_directory=None,
|
||||
huge_pages=False, use_raylet=False):
|
||||
def start_objstore(node_ip_address,
|
||||
redis_address,
|
||||
object_manager_port=None,
|
||||
store_stdout_file=None,
|
||||
store_stderr_file=None,
|
||||
manager_stdout_file=None,
|
||||
manager_stderr_file=None,
|
||||
objstore_memory=None,
|
||||
cleanup=True,
|
||||
plasma_directory=None,
|
||||
huge_pages=False,
|
||||
use_raylet=False):
|
||||
"""This method starts an object store process.
|
||||
|
||||
Args:
|
||||
@@ -1013,24 +1027,24 @@ def start_objstore(node_ip_address, redis_address,
|
||||
if object_manager_port is not None:
|
||||
(plasma_manager_name, p2,
|
||||
plasma_manager_port) = ray.plasma.start_plasma_manager(
|
||||
plasma_store_name,
|
||||
redis_address,
|
||||
plasma_manager_port=object_manager_port,
|
||||
node_ip_address=node_ip_address,
|
||||
num_retries=1,
|
||||
run_profiler=RUN_PLASMA_MANAGER_PROFILER,
|
||||
stdout_file=manager_stdout_file,
|
||||
stderr_file=manager_stderr_file)
|
||||
plasma_store_name,
|
||||
redis_address,
|
||||
plasma_manager_port=object_manager_port,
|
||||
node_ip_address=node_ip_address,
|
||||
num_retries=1,
|
||||
run_profiler=RUN_PLASMA_MANAGER_PROFILER,
|
||||
stdout_file=manager_stdout_file,
|
||||
stderr_file=manager_stderr_file)
|
||||
assert plasma_manager_port == object_manager_port
|
||||
else:
|
||||
(plasma_manager_name, p2,
|
||||
plasma_manager_port) = ray.plasma.start_plasma_manager(
|
||||
plasma_store_name,
|
||||
redis_address,
|
||||
node_ip_address=node_ip_address,
|
||||
run_profiler=RUN_PLASMA_MANAGER_PROFILER,
|
||||
stdout_file=manager_stdout_file,
|
||||
stderr_file=manager_stderr_file)
|
||||
plasma_store_name,
|
||||
redis_address,
|
||||
node_ip_address=node_ip_address,
|
||||
run_profiler=RUN_PLASMA_MANAGER_PROFILER,
|
||||
stdout_file=manager_stdout_file,
|
||||
stderr_file=manager_stderr_file)
|
||||
else:
|
||||
plasma_manager_port = None
|
||||
plasma_manager_name = None
|
||||
@@ -1049,9 +1063,15 @@ def start_objstore(node_ip_address, redis_address,
|
||||
plasma_manager_port)
|
||||
|
||||
|
||||
def start_worker(node_ip_address, object_store_name, object_store_manager_name,
|
||||
local_scheduler_name, redis_address, worker_path,
|
||||
stdout_file=None, stderr_file=None, cleanup=True):
|
||||
def start_worker(node_ip_address,
|
||||
object_store_name,
|
||||
object_store_manager_name,
|
||||
local_scheduler_name,
|
||||
redis_address,
|
||||
worker_path,
|
||||
stdout_file=None,
|
||||
stderr_file=None,
|
||||
cleanup=True):
|
||||
"""This method starts a worker process.
|
||||
|
||||
Args:
|
||||
@@ -1072,14 +1092,14 @@ def start_worker(node_ip_address, object_store_name, object_store_manager_name,
|
||||
Python process that imported services exits. This is True by
|
||||
default.
|
||||
"""
|
||||
command = [sys.executable,
|
||||
"-u",
|
||||
worker_path,
|
||||
"--node-ip-address=" + node_ip_address,
|
||||
"--object-store-name=" + object_store_name,
|
||||
"--object-store-manager-name=" + object_store_manager_name,
|
||||
"--local-scheduler-name=" + local_scheduler_name,
|
||||
"--redis-address=" + str(redis_address)]
|
||||
command = [
|
||||
sys.executable, "-u", worker_path,
|
||||
"--node-ip-address=" + node_ip_address,
|
||||
"--object-store-name=" + object_store_name,
|
||||
"--object-store-manager-name=" + object_store_manager_name,
|
||||
"--local-scheduler-name=" + local_scheduler_name,
|
||||
"--redis-address=" + str(redis_address)
|
||||
]
|
||||
p = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file)
|
||||
if cleanup:
|
||||
all_processes[PROCESS_TYPE_WORKER].append(p)
|
||||
@@ -1087,8 +1107,12 @@ def start_worker(node_ip_address, object_store_name, object_store_manager_name,
|
||||
[stdout_file, stderr_file])
|
||||
|
||||
|
||||
def start_monitor(redis_address, node_ip_address, stdout_file=None,
|
||||
stderr_file=None, cleanup=True, autoscaling_config=None):
|
||||
def start_monitor(redis_address,
|
||||
node_ip_address,
|
||||
stdout_file=None,
|
||||
stderr_file=None,
|
||||
cleanup=True,
|
||||
autoscaling_config=None):
|
||||
"""Run a process to monitor the other processes.
|
||||
|
||||
Args:
|
||||
@@ -1105,12 +1129,12 @@ def start_monitor(redis_address, node_ip_address, stdout_file=None,
|
||||
default.
|
||||
autoscaling_config: path to autoscaling config file.
|
||||
"""
|
||||
monitor_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||||
"monitor.py")
|
||||
command = [sys.executable,
|
||||
"-u",
|
||||
monitor_path,
|
||||
"--redis-address=" + str(redis_address)]
|
||||
monitor_path = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)), "monitor.py")
|
||||
command = [
|
||||
sys.executable, "-u", monitor_path,
|
||||
"--redis-address=" + str(redis_address)
|
||||
]
|
||||
if autoscaling_config:
|
||||
command.append("--autoscaling-config=" + str(autoscaling_config))
|
||||
p = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file)
|
||||
@@ -1120,8 +1144,10 @@ def start_monitor(redis_address, node_ip_address, stdout_file=None,
|
||||
[stdout_file, stderr_file])
|
||||
|
||||
|
||||
def start_raylet_monitor(redis_address, stdout_file=None,
|
||||
stderr_file=None, cleanup=True):
|
||||
def start_raylet_monitor(redis_address,
|
||||
stdout_file=None,
|
||||
stderr_file=None,
|
||||
cleanup=True):
|
||||
"""Run a process to monitor the other processes.
|
||||
|
||||
Args:
|
||||
@@ -1136,9 +1162,7 @@ def start_raylet_monitor(redis_address, stdout_file=None,
|
||||
default.
|
||||
"""
|
||||
gcs_ip_address, gcs_port = redis_address.split(":")
|
||||
command = [RAYLET_MONITOR_EXECUTABLE,
|
||||
gcs_ip_address,
|
||||
gcs_port]
|
||||
command = [RAYLET_MONITOR_EXECUTABLE, gcs_ip_address, gcs_port]
|
||||
p = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file)
|
||||
if cleanup:
|
||||
all_processes[PROCESS_TYPE_MONITOR].append(p)
|
||||
@@ -1238,16 +1262,17 @@ def start_ray_processes(address_info=None,
|
||||
workers_per_local_scheduler = []
|
||||
for resource_dict in resources:
|
||||
cpus = resource_dict.get("CPU")
|
||||
workers_per_local_scheduler.append(cpus if cpus is not None
|
||||
else psutil.cpu_count())
|
||||
workers_per_local_scheduler.append(cpus if cpus is not None else
|
||||
psutil.cpu_count())
|
||||
|
||||
if address_info is None:
|
||||
address_info = {}
|
||||
address_info["node_ip_address"] = node_ip_address
|
||||
|
||||
if worker_path is None:
|
||||
worker_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||||
"workers/default_worker.py")
|
||||
worker_path = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)),
|
||||
"workers/default_worker.py")
|
||||
|
||||
# Start Redis if there isn't already an instance running. TODO(rkn): We are
|
||||
# suppressing the output of Redis because on Linux it prints a bunch of
|
||||
@@ -1257,7 +1282,8 @@ def start_ray_processes(address_info=None,
|
||||
redis_shards = address_info.get("redis_shards", [])
|
||||
if redis_address is None:
|
||||
redis_address, redis_shards = start_redis(
|
||||
node_ip_address, port=redis_port,
|
||||
node_ip_address,
|
||||
port=redis_port,
|
||||
redis_shard_ports=redis_shard_ports,
|
||||
num_redis_shards=num_redis_shards,
|
||||
redis_max_clients=redis_max_clients,
|
||||
@@ -1274,23 +1300,25 @@ def start_ray_processes(address_info=None,
|
||||
# Start monitoring the processes.
|
||||
monitor_stdout_file, monitor_stderr_file = new_log_files(
|
||||
"monitor", redirect_output)
|
||||
start_monitor(redis_address,
|
||||
node_ip_address,
|
||||
stdout_file=monitor_stdout_file,
|
||||
stderr_file=monitor_stderr_file,
|
||||
cleanup=cleanup,
|
||||
autoscaling_config=autoscaling_config)
|
||||
start_monitor(
|
||||
redis_address,
|
||||
node_ip_address,
|
||||
stdout_file=monitor_stdout_file,
|
||||
stderr_file=monitor_stderr_file,
|
||||
cleanup=cleanup,
|
||||
autoscaling_config=autoscaling_config)
|
||||
if use_raylet:
|
||||
start_raylet_monitor(redis_address,
|
||||
stdout_file=monitor_stdout_file,
|
||||
stderr_file=monitor_stderr_file,
|
||||
cleanup=cleanup)
|
||||
start_raylet_monitor(
|
||||
redis_address,
|
||||
stdout_file=monitor_stdout_file,
|
||||
stderr_file=monitor_stderr_file,
|
||||
cleanup=cleanup)
|
||||
|
||||
if redis_shards == []:
|
||||
# Get redis shards from primary redis instance.
|
||||
redis_ip_address, redis_port = redis_address.split(":")
|
||||
redis_client = redis.StrictRedis(host=redis_ip_address,
|
||||
port=redis_port)
|
||||
redis_client = redis.StrictRedis(
|
||||
host=redis_ip_address, port=redis_port)
|
||||
redis_shards = redis_client.lrange("RedisShards", start=0, end=-1)
|
||||
redis_shards = [shard.decode("ascii") for shard in redis_shards]
|
||||
address_info["redis_shards"] = redis_shards
|
||||
@@ -1299,21 +1327,23 @@ def start_ray_processes(address_info=None,
|
||||
if include_log_monitor:
|
||||
log_monitor_stdout_file, log_monitor_stderr_file = new_log_files(
|
||||
"log_monitor", redirect_output=True)
|
||||
start_log_monitor(redis_address,
|
||||
node_ip_address,
|
||||
stdout_file=log_monitor_stdout_file,
|
||||
stderr_file=log_monitor_stderr_file,
|
||||
cleanup=cleanup)
|
||||
start_log_monitor(
|
||||
redis_address,
|
||||
node_ip_address,
|
||||
stdout_file=log_monitor_stdout_file,
|
||||
stderr_file=log_monitor_stderr_file,
|
||||
cleanup=cleanup)
|
||||
|
||||
# Start the global scheduler, if necessary.
|
||||
if include_global_scheduler and not use_raylet:
|
||||
global_scheduler_stdout_file, global_scheduler_stderr_file = (
|
||||
new_log_files("global_scheduler", redirect_output))
|
||||
start_global_scheduler(redis_address,
|
||||
node_ip_address,
|
||||
stdout_file=global_scheduler_stdout_file,
|
||||
stderr_file=global_scheduler_stderr_file,
|
||||
cleanup=cleanup)
|
||||
start_global_scheduler(
|
||||
redis_address,
|
||||
node_ip_address,
|
||||
stdout_file=global_scheduler_stdout_file,
|
||||
stderr_file=global_scheduler_stderr_file,
|
||||
cleanup=cleanup)
|
||||
|
||||
# Initialize with existing services.
|
||||
if "object_store_addresses" not in address_info:
|
||||
@@ -1324,9 +1354,8 @@ def start_ray_processes(address_info=None,
|
||||
local_scheduler_socket_names = address_info["local_scheduler_socket_names"]
|
||||
|
||||
# Get the ports to use for the object managers if any are provided.
|
||||
object_manager_ports = (address_info["object_manager_ports"]
|
||||
if "object_manager_ports" in address_info
|
||||
else None)
|
||||
object_manager_ports = (address_info["object_manager_ports"] if
|
||||
"object_manager_ports" in address_info else None)
|
||||
if not isinstance(object_manager_ports, list):
|
||||
object_manager_ports = num_local_schedulers * [object_manager_ports]
|
||||
assert len(object_manager_ports) == num_local_schedulers
|
||||
@@ -1347,7 +1376,8 @@ def start_ray_processes(address_info=None,
|
||||
manager_stdout_file=plasma_manager_stdout_file,
|
||||
manager_stderr_file=plasma_manager_stderr_file,
|
||||
objstore_memory=object_store_memory,
|
||||
cleanup=cleanup, plasma_directory=plasma_directory,
|
||||
cleanup=cleanup,
|
||||
plasma_directory=plasma_directory,
|
||||
huge_pages=huge_pages,
|
||||
use_raylet=use_raylet)
|
||||
object_store_addresses.append(object_store_address)
|
||||
@@ -1355,8 +1385,8 @@ def start_ray_processes(address_info=None,
|
||||
|
||||
# Start any local schedulers that do not yet exist.
|
||||
if not use_raylet:
|
||||
for i in range(len(local_scheduler_socket_names),
|
||||
num_local_schedulers):
|
||||
for i in range(
|
||||
len(local_scheduler_socket_names), num_local_schedulers):
|
||||
# Connect the local scheduler to the object store at the same
|
||||
# index.
|
||||
object_store_address = object_store_addresses[i]
|
||||
@@ -1374,8 +1404,9 @@ def start_ray_processes(address_info=None,
|
||||
# redirect the worker output, then we cannot redirect the local
|
||||
# scheduler output.
|
||||
local_scheduler_stdout_file, local_scheduler_stderr_file = (
|
||||
new_log_files("local_scheduler_{}".format(i),
|
||||
redirect_output=redirect_worker_output))
|
||||
new_log_files(
|
||||
"local_scheduler_{}".format(i),
|
||||
redirect_output=redirect_worker_output))
|
||||
local_scheduler_name = start_local_scheduler(
|
||||
redis_address,
|
||||
node_ip_address,
|
||||
@@ -1398,17 +1429,18 @@ def start_ray_processes(address_info=None,
|
||||
else:
|
||||
# Start the raylet. TODO(rkn): Modify this to allow starting
|
||||
# multiple raylets on the same machine.
|
||||
raylet_stdout_file, raylet_stderr_file = (
|
||||
new_log_files("raylet_{}".format(i),
|
||||
redirect_output=redirect_output))
|
||||
address_info["raylet_socket_names"] = [start_raylet(
|
||||
redis_address,
|
||||
node_ip_address,
|
||||
object_store_addresses[i].name,
|
||||
worker_path,
|
||||
stdout_file=None,
|
||||
stderr_file=None,
|
||||
cleanup=cleanup)]
|
||||
raylet_stdout_file, raylet_stderr_file = (new_log_files(
|
||||
"raylet_{}".format(i), redirect_output=redirect_output))
|
||||
address_info["raylet_socket_names"] = [
|
||||
start_raylet(
|
||||
redis_address,
|
||||
node_ip_address,
|
||||
object_store_addresses[i].name,
|
||||
worker_path,
|
||||
stdout_file=None,
|
||||
stderr_file=None,
|
||||
cleanup=cleanup)
|
||||
]
|
||||
|
||||
if not use_raylet:
|
||||
# Start any workers that the local scheduler has not already started.
|
||||
@@ -1419,28 +1451,30 @@ def start_ray_processes(address_info=None,
|
||||
for j in range(num_local_scheduler_workers):
|
||||
worker_stdout_file, worker_stderr_file = new_log_files(
|
||||
"worker_{}_{}".format(i, j), redirect_output)
|
||||
start_worker(node_ip_address,
|
||||
object_store_address.name,
|
||||
object_store_address.manager_name,
|
||||
local_scheduler_name,
|
||||
redis_address,
|
||||
worker_path,
|
||||
stdout_file=worker_stdout_file,
|
||||
stderr_file=worker_stderr_file,
|
||||
cleanup=cleanup)
|
||||
start_worker(
|
||||
node_ip_address,
|
||||
object_store_address.name,
|
||||
object_store_address.manager_name,
|
||||
local_scheduler_name,
|
||||
redis_address,
|
||||
worker_path,
|
||||
stdout_file=worker_stdout_file,
|
||||
stderr_file=worker_stderr_file,
|
||||
cleanup=cleanup)
|
||||
workers_per_local_scheduler[i] -= 1
|
||||
|
||||
# Make sure that we've started all the workers.
|
||||
assert(sum(workers_per_local_scheduler) == 0)
|
||||
assert (sum(workers_per_local_scheduler) == 0)
|
||||
|
||||
# Try to start the web UI.
|
||||
if include_webui:
|
||||
ui_stdout_file, ui_stderr_file = new_log_files(
|
||||
"webui", redirect_output=True)
|
||||
address_info["webui_url"] = start_ui(redis_address,
|
||||
stdout_file=ui_stdout_file,
|
||||
stderr_file=ui_stderr_file,
|
||||
cleanup=cleanup)
|
||||
address_info["webui_url"] = start_ui(
|
||||
redis_address,
|
||||
stdout_file=ui_stdout_file,
|
||||
stderr_file=ui_stderr_file,
|
||||
cleanup=cleanup)
|
||||
else:
|
||||
address_info["webui_url"] = ""
|
||||
# Return the addresses of the relevant processes.
|
||||
@@ -1500,21 +1534,24 @@ def start_ray_node(node_ip_address,
|
||||
A dictionary of the address information for the processes that were
|
||||
started.
|
||||
"""
|
||||
address_info = {"redis_address": redis_address,
|
||||
"object_manager_ports": object_manager_ports}
|
||||
return start_ray_processes(address_info=address_info,
|
||||
node_ip_address=node_ip_address,
|
||||
num_workers=num_workers,
|
||||
num_local_schedulers=num_local_schedulers,
|
||||
object_store_memory=object_store_memory,
|
||||
worker_path=worker_path,
|
||||
include_log_monitor=True,
|
||||
cleanup=cleanup,
|
||||
redirect_worker_output=redirect_worker_output,
|
||||
redirect_output=redirect_output,
|
||||
resources=resources,
|
||||
plasma_directory=plasma_directory,
|
||||
huge_pages=huge_pages)
|
||||
address_info = {
|
||||
"redis_address": redis_address,
|
||||
"object_manager_ports": object_manager_ports
|
||||
}
|
||||
return start_ray_processes(
|
||||
address_info=address_info,
|
||||
node_ip_address=node_ip_address,
|
||||
num_workers=num_workers,
|
||||
num_local_schedulers=num_local_schedulers,
|
||||
object_store_memory=object_store_memory,
|
||||
worker_path=worker_path,
|
||||
include_log_monitor=True,
|
||||
cleanup=cleanup,
|
||||
redirect_worker_output=redirect_worker_output,
|
||||
redirect_output=redirect_output,
|
||||
resources=resources,
|
||||
plasma_directory=plasma_directory,
|
||||
huge_pages=huge_pages)
|
||||
|
||||
|
||||
def start_ray_head(address_info=None,
|
||||
|
||||
Reference in New Issue
Block a user