wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-19 11:27:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add warning/error if object store memory exceeds available memory (#4893)

Browse Source 
* exclude

* format

* add warning

* hatch

* reduce mem usage

* reduce object store mem

* set obj mem
This commit is contained in:
Eric Liang
2019-07-08 21:37:08 -07:00
committed by GitHub
parent dfc94ce7bc
commit 5aec750107
6 changed files with 73 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/services.py
+26
View File
@@ -1300,6 +1300,32 @@ def determine_plasma_store_config(object_store_memory=None,
object_store_memory = (
ray_constants.DEFAULT_OBJECT_STORE_MAX_MEMORY_BYTES)
# Other applications may also be using a lot of memory on the same
# node. Try to detect when this is happening and log a warning or
# error in more severe cases.
avail_memory = ray.utils.estimate_available_memory()
object_store_fraction = object_store_memory / avail_memory
# Escape hatch, undocumented for now.
no_check = os.environ.get("RAY_DEBUG_DISABLE_MEM_CHECKS", False)
if object_store_fraction > 0.9 and not no_check:
raise ValueError(
"The default object store size of {} GB "
"will use more than 90% of the available memory on this node "
"({} GB). Please reduce the object store memory size "
"to avoid memory contention with other applications, or "
"shut down the applications using this memory.".format(
round(object_store_memory / 1e9, 2),
round(avail_memory / 1e9, 2)))
elif object_store_fraction > 0.5:
logger.warning(
"WARNING: The default object store size of {} GB "
"will use more than 50% of the available memory on this node "
"({} GB). Consider setting the object store memory manually "
"to a smaller size to avoid memory contention with other "
"applications.".format(
round(object_store_memory / 1e9, 2),
round(avail_memory / 1e9, 2)))
# Determine which directory to use. By default, use /tmp on MacOS and
# /dev/shm on Linux, unless the shared-memory file system is too small,
# in which case we default to /tmp on Linux.