wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-12 19:49:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add option to evict keys LRU from the sharded redis tables (#3499)

Browse Source 
* wip

* wip

* format

* wip

* note

* lint

* fix

* flag

* typo

* raise timeout

* fix

* optional get

* fix flag

* increase timeout in test

* update docs

* format
This commit is contained in:
Eric Liang
2018-12-09 05:48:52 -08:00
committed by Philipp Moritz
parent 0136af5aac
commit cffe8f9806
11 changed files with 174 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/worker.py
+38 -4
View File
@@ -217,7 +217,7 @@ class Worker(object):
# When the worker is constructed. Record the original value of the
# CUDA_VISIBLE_DEVICES environment variable.
self.original_gpu_ids = ray.utils.get_cuda_visible_devices()
self.profiler = profiling.Profiler(self)
self.profiler = None
self.memory_monitor = memory_monitor.MemoryMonitor()
self.state_lock = threading.Lock()
# A dictionary that maps from driver id to SerializationContext
@@ -1342,6 +1342,8 @@ def _init(address_info=None,
num_workers=None,
num_local_schedulers=None,
object_store_memory=None,
redis_max_memory=None,
collect_profiling_data=True,
local_mode=False,
driver_mode=None,
redirect_worker_output=False,
@@ -1386,6 +1388,11 @@ def _init(address_info=None,
This is only provided if start_ray_local is True.
object_store_memory: The maximum amount of memory (in bytes) to
allow the object store to use.
redis_max_memory: The max amount of memory (in bytes) to allow redis
to use, or None for no limit. Once the limit is exceeded, redis
will start LRU eviction of entries. This only applies to the
sharded redis tables (task and object tables).
collect_profiling_data: Whether to collect profiling data from workers.
local_mode (bool): True if the code should be executed serially
without Ray. This is useful for debugging.
redirect_worker_output: True if the stdout and stderr of worker
@@ -1440,6 +1447,11 @@ def _init(address_info=None,
else:
driver_mode = SCRIPT_MODE
if redis_max_memory and collect_profiling_data:
logger.warn("Profiling data cannot be LRU evicted, so it is disabled "
"when redis_max_memory is set.")
collect_profiling_data = False
# Get addresses of existing services.
if address_info is None:
address_info = {}
@@ -1479,6 +1491,8 @@ def _init(address_info=None,
num_workers=num_workers,
num_local_schedulers=num_local_schedulers,
object_store_memory=object_store_memory,
redis_max_memory=redis_max_memory,
collect_profiling_data=collect_profiling_data,
redirect_worker_output=redirect_worker_output,
redirect_output=redirect_output,
start_workers_from_local_scheduler=(
@@ -1519,6 +1533,9 @@ def _init(address_info=None,
if object_store_memory is not None:
raise Exception("When connecting to an existing cluster, "
"object_store_memory must not be provided.")
if redis_max_memory is not None:
raise Exception("When connecting to an existing cluster, "
"redis_max_memory must not be provided.")
if plasma_directory is not None:
raise Exception("When connecting to an existing cluster, "
"plasma_directory must not be provided.")
@@ -1556,7 +1573,7 @@ def _init(address_info=None,
"node_ip_address": node_ip_address,
"redis_address": address_info["redis_address"],
"store_socket_name": address_info["object_store_addresses"][0],
"webui_url": address_info["webui_url"]
"webui_url": address_info["webui_url"],
}
driver_address_info["raylet_socket_name"] = (
address_info["raylet_socket_names"][0])
@@ -1569,7 +1586,8 @@ def _init(address_info=None,
mode=driver_mode,
worker=global_worker,
driver_id=driver_id,
redis_password=redis_password)
redis_password=redis_password,
collect_profiling_data=collect_profiling_data)
return address_info
@@ -1578,6 +1596,8 @@ def init(redis_address=None,
num_gpus=None,
resources=None,
object_store_memory=None,
redis_max_memory=None,
collect_profiling_data=True,
node_ip_address=None,
object_id_seed=None,
num_workers=None,
@@ -1634,6 +1654,11 @@ def init(redis_address=None,
of that resource available.
object_store_memory: The amount of memory (in bytes) to start the
object store with.
redis_max_memory: The max amount of memory (in bytes) to allow redis
to use, or None for no limit. Once the limit is exceeded, redis
will start LRU eviction of entries. This only applies to the
sharded redis tables (task and object tables).
collect_profiling_data: Whether to collect profiling data from workers.
node_ip_address (str): The IP address of the node that we are on.
object_id_seed (int): Used to seed the deterministic generation of
object IDs. The same value can be used across multiple runs of the
@@ -1734,6 +1759,8 @@ def init(redis_address=None,
huge_pages=huge_pages,
include_webui=include_webui,
object_store_memory=object_store_memory,
redis_max_memory=redis_max_memory,
collect_profiling_data=collect_profiling_data,
driver_id=driver_id,
plasma_store_socket_name=plasma_store_socket_name,
raylet_socket_name=raylet_socket_name,
@@ -1913,7 +1940,8 @@ def connect(info,
mode=WORKER_MODE,
worker=global_worker,
driver_id=None,
redis_password=None):
redis_password=None,
collect_profiling_data=True):
"""Connect this worker to the local scheduler, to Plasma, and to Redis.
Args:
@@ -1926,6 +1954,7 @@ def connect(info,
driver_id: The ID of driver. If it's None, then we will generate one.
redis_password (str): Prevents external clients without the password
from connecting to Redis if provided.
collect_profiling_data: Whether to collect profiling data from workers.
"""
# Do some basic checking to make sure we didn't call ray.init twice.
error_message = "Perhaps you called ray.init twice by accident?"
@@ -1935,6 +1964,11 @@ def connect(info,
# Enable nice stack traces on SIGSEGV etc.
faulthandler.enable(all_threads=False)
if collect_profiling_data:
worker.profiler = profiling.Profiler(worker)
else:
worker.profiler = profiling.NoopProfiler()
# Initialize some fields.
if mode is WORKER_MODE:
worker.worker_id = random_string()