wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-27 22:53:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Core] Allow users to specify the classpath and import path (#10560)

Browse Source 
* move job resource path to job config

* job resource path support list

* job resource path support for python

* fix job_resource_path support

* fix worker command

* fix job config

* use jar file instead of parent path

* fix job resource path

* add test to test.sh

* lint

* Update java/runtime/src/main/resources/ray.default.conf

Co-authored-by: Kai Yang <kfstorm@outlook.com>

* fix testGetFunctionFromLocalResource

* lint

* fix rebase

* add jars in resource path to classloader

* add job_resource_path to worker

* add ray stop

* rename job_resource_path to resource_path

* fix resource_path

* refine resource_path comments

* rename job resource path to code search path

* Add instruction about starting a cross-language cluster

* fix ClassLoaderTest.java

* add code-search-path to RunManager

* refine comments for code-search-path

* rename resourcePath to codeSearchPath

* Update doc

* fix

* rename resourcePath to codeSearchPath

* update doc

* filter out empty path

* fix comments

* fix comments

* fix tests

* revert pom

* lint

* fix doc

* update doc

* Apply suggestions from code review

* lint

Co-authored-by: Kai Yang <kfstorm@outlook.com>
Co-authored-by: Hao Chen <chenh1024@gmail.com>
This commit is contained in:
chaokunyang
2020-09-09 00:46:32 +08:00
committed by GitHub
parent 3645a05644
commit bbfbc98a41
22 changed files with 226 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/job_config.py
+7
View File
@@ -10,6 +10,9 @@ class JobConfig:
num_java_workers_per_process (int): The number of java workers per
worker process.
jvm_options (str[]): The jvm options for java workers of the job.
code_search_path (list): A list of directories or jar files that
specify the search path for user code. This will be used as
`CLASSPATH` in Java and `PYTHONPATH` in Python.
"""
def __init__(
@@ -17,6 +20,7 @@ class JobConfig:
worker_env=None,
num_java_workers_per_process=1,
jvm_options=None,
code_search_path=None,
):
if worker_env is None:
self.worker_env = dict()
@@ -27,6 +31,8 @@ class JobConfig:
self.jvm_options = []
else:
self.jvm_options = jvm_options
if code_search_path is None:
self.code_search_path = []
def serialize(self):
job_config = ray.gcs_utils.JobConfig()
@@ -35,4 +41,5 @@ class JobConfig:
job_config.num_java_workers_per_process = (
self.num_java_workers_per_process)
job_config.jvm_options.extend(self.jvm_options)
job_config.code_search_path.extend(self.code_search_path)
return job_config.SerializeToString()
python/ray/node.py
+2 -1
View File
@@ -730,7 +730,8 @@ class Node:
head_node=self.head,
start_initial_python_workers_for_first_job=self._ray_params.
start_initial_python_workers_for_first_job,
object_spilling_config=self._ray_params.object_spilling_config)
object_spilling_config=self._ray_params.object_spilling_config,
code_search_path=self._ray_params.code_search_path)
assert ray_constants.PROCESS_TYPE_RAYLET not in self.all_processes
self.all_processes[ray_constants.PROCESS_TYPE_RAYLET] = [process_info]
python/ray/parameter.py
+5 -1
View File
@@ -147,7 +147,8 @@ class RayParams:
metrics_agent_port=None,
metrics_export_port=None,
lru_evict=False,
object_spilling_config=None):
object_spilling_config=None,
code_search_path=None):
self.object_ref_seed = object_ref_seed
self.redis_address = redis_address
self.num_cpus = num_cpus
@@ -194,6 +195,9 @@ class RayParams:
self._enable_object_reconstruction = enable_object_reconstruction
self.object_spilling_config = object_spilling_config
self._check_usage()
self.code_search_path = code_search_path
if code_search_path is None:
self.code_search_path = []
# Set the internal config options for LRU eviction.
if lru_evict:
python/ray/scripts/scripts.py
+11 -3
View File
@@ -354,6 +354,13 @@ def dashboard(cluster_config_file, cluster_name, port, remote_port):
default=None,
type=str,
help="Overwrite the options to start Java workers.")
@click.option(
"--code-search-path",
default=None,
type=str,
help="A list of directories or jar files separated by colon that specify "
"the search path for user code. This will be used as `CLASSPATH` in "
"Java and `PYTHONPATH` in Python.")
@click.option(
"--system-config",
default=None,
@@ -391,9 +398,9 @@ def start(node_ip_address, redis_address, address, redis_port, port,
dashboard_port, block, plasma_directory, huge_pages,
autoscaling_config, no_redirect_worker_output, no_redirect_output,
plasma_store_socket_name, raylet_socket_name, temp_dir, include_java,
java_worker_options, load_code_from_local, system_config, lru_evict,
enable_object_reconstruction, metrics_export_port, log_style,
log_color, verbose):
java_worker_options, code_search_path, load_code_from_local,
system_config, lru_evict, enable_object_reconstruction,
metrics_export_port, log_style, log_color, verbose):
"""Start Ray processes manually on the local machine."""
cli_logger.log_style = log_style
cli_logger.color_mode = log_color
@@ -504,6 +511,7 @@ def start(node_ip_address, redis_address, address, redis_port, port,
dashboard_port=dashboard_port,
java_worker_options=java_worker_options,
load_code_from_local=load_code_from_local,
code_search_path=code_search_path,
_system_config=system_config,
lru_evict=lru_evict,
enable_object_reconstruction=enable_object_reconstruction,
python/ray/services.py
+21 -23
View File
@@ -11,16 +11,15 @@ import socket
import subprocess
import sys
import time
import redis
import colorama
import colorful as cf
import psutil
# Ray modules
import ray
import ray.ray_constants as ray_constants
import psutil
import redis
from ray.autoscaler.cli_logger import cli_logger
import colorful as cf
resource = None
if sys.platform != "win32":
@@ -71,11 +70,6 @@ DEFAULT_WORKER_EXECUTABLE = os.path.join(
os.path.abspath(os.path.dirname(__file__)),
"core/src/ray/cpp/default_worker" + EXE_SUFFIX)
DEFAULT_JAVA_WORKER_CLASSPATH = [
os.path.join(
os.path.abspath(os.path.dirname(__file__)), "../../../build/java/*"),
]
# Logger for this module. It should be configured at the entry point
# into the program using Ray. Ray provides a default configuration at
# entry/init points.
@@ -1282,7 +1276,8 @@ def start_raylet(redis_address,
socket_to_use=None,
head_node=False,
start_initial_python_workers_for_first_job=False,
object_spilling_config=None):
object_spilling_config=None,
code_search_path=None):
"""Start a raylet, which is a combined local scheduler and object manager.
Args:
@@ -1320,6 +1315,9 @@ def start_raylet(redis_address,
include_java (bool): If True, the raylet backend can also support
Java worker.
java_worker_options (list): The command options for Java worker.
code_search_path (list): Code search path for worker. code_search_path
is added to worker command in non-multi-tenancy mode and job_config
in multi-tenancy mode.
Returns:
ProcessInfo for the process that was started.
"""
@@ -1348,16 +1346,15 @@ def start_raylet(redis_address,
gcs_ip_address, gcs_port = redis_address.split(":")
if include_java is True:
default_cp = os.pathsep.join(DEFAULT_JAVA_WORKER_CLASSPATH)
java_worker_command = build_java_worker_command(
json.loads(java_worker_options)
if java_worker_options else ["-classpath", default_cp],
json.loads(java_worker_options) if java_worker_options else [],
redis_address,
node_manager_port,
plasma_store_name,
raylet_name,
redis_password,
session_dir,
code_search_path,
)
else:
java_worker_command = []
@@ -1384,6 +1381,8 @@ def start_raylet(redis_address,
f"--config-list={config_str}", f"--temp-dir={temp_dir}",
f"--metrics-agent-port={metrics_agent_port}"
]
if code_search_path:
start_worker_command.append(f"--code-search-path={code_search_path}")
if redis_password:
start_worker_command += [f"--redis-password={redis_password}"]
@@ -1398,6 +1397,9 @@ def start_raylet(redis_address,
if max_worker_port is None:
max_worker_port = 0
if code_search_path is not None and len(code_search_path) > 0:
load_code_from_local = True
if load_code_from_local:
start_worker_command += ["--load-code-from-local"]
@@ -1493,15 +1495,10 @@ def get_ray_jars_dir():
return os.path.abspath(os.path.join(current_dir, "jars"))
def build_java_worker_command(
java_worker_options,
redis_address,
node_manager_port,
plasma_store_name,
raylet_name,
redis_password,
session_dir,
):
def build_java_worker_command(java_worker_options, redis_address,
node_manager_port, plasma_store_name,
raylet_name, redis_password, session_dir,
code_search_path):
"""This method assembles the command used to start a Java worker.
Args:
@@ -1512,6 +1509,7 @@ def build_java_worker_command(
raylet_name (str): The name of the raylet socket to create.
redis_password (str): The password of connect to redis.
session_dir (str): The path of this session.
code_search_path (list): Teh job code search path.
Returns:
The command string for starting Java worker.
"""
@@ -1532,7 +1530,7 @@ def build_java_worker_command(
pairs.append(("ray.home", RAY_HOME))
pairs.append(("ray.logging.dir", os.path.join(session_dir, "logs")))
pairs.append(("ray.session-dir", session_dir))
pairs.append(("ray.job.code-search-path", code_search_path))
command = ["java"] + ["-D{}={}".format(*pair) for pair in pairs]
command += ["RAY_WORKER_RAYLET_CONFIG_PLACEHOLDER"]
python/ray/worker.py
+3
View File
@@ -493,6 +493,7 @@ def init(
_redis_password=ray_constants.REDIS_DEFAULT_PASSWORD,
_include_java=False,
_java_worker_options=None,
_code_search_path=None,
_temp_dir=None,
_load_code_from_local=False,
_lru_evict=False,
@@ -587,6 +588,7 @@ def init(
_load_code_from_local: Whether code should be loaded from a local
module or from the GCS.
_java_worker_options: Overwrite the options to start Java workers.
_code_search_path (list): Java classpath or python import path.
_lru_evict (bool): If True, when an object store is full, it will evict
objects in LRU order to make more space and when under memory
pressure, ray.ObjectLostError may be thrown. If False, then
@@ -682,6 +684,7 @@ def init(
temp_dir=_temp_dir,
load_code_from_local=_load_code_from_local,
java_worker_options=_java_worker_options,
code_search_path=_code_search_path,
start_initial_python_workers_for_first_job=True,
_system_config=_system_config,
lru_evict=_lru_evict,
python/ray/workers/default_worker.py
+16 -1
View File
@@ -1,6 +1,8 @@
import argparse
import json
import time
import sys
import os
import ray
import ray.actor
@@ -98,7 +100,13 @@ parser.add_argument(
type=str,
default="",
help="The configuration of object spilling. Only used by I/O workers.")
parser.add_argument(
"--code-search-path",
default=None,
type=str,
help="A list of directories or jar files separated by colon that specify "
"the search path for user code. This will be used as `CLASSPATH` in "
"Java and `PYTHONPATH` in Python.")
if __name__ == "__main__":
args = parser.parse_args()
@@ -135,6 +143,13 @@ if __name__ == "__main__":
if raylet_ip_address is None:
raylet_ip_address = args.node_ip_address
code_search_path = args.code_search_path
if code_search_path is not None:
for p in code_search_path.split(":"):
if os.path.isfile(p):
p = os.path.dirname(p)
sys.path.append(p)
ray_params = RayParams(
node_ip_address=args.node_ip_address,
raylet_ip_address=raylet_ip_address,