wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-08 18:24:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add warning/error if object store memory exceeds available memory (#4893)

Browse Source 
* exclude

* format

* add warning

* hatch

* reduce mem usage

* reduce object store mem

* set obj mem
This commit is contained in:
Eric Liang
2019-07-08 21:37:08 -07:00
committed by GitHub
parent dfc94ce7bc
commit 5aec750107
6 changed files with 73 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/memory_monitor.py
+10 -5
View File
@@ -24,21 +24,26 @@ class RayOutOfMemoryError(Exception):
proc_stats = []
for pid in pids:
proc = psutil.Process(pid)
proc_stats.append((proc.memory_info().rss, pid, proc.cmdline()))
proc_stats.append(
(proc.memory_info().rss - proc.memory_info().shared, pid,
proc.cmdline()))
proc_str = "PID\tMEM\tCOMMAND"
for rss, pid, cmdline in sorted(proc_stats, reverse=True)[:5]:
for rss, pid, cmdline in sorted(proc_stats, reverse=True)[:10]:
proc_str += "\n{}\t{}GB\t{}".format(
pid, round(rss / 1e9, 2), " ".join(cmdline)[:100].strip())
return ("More than {}% of the memory on ".format(int(
100 * threshold)) + "node {} is used ({} / {} GB). ".format(
os.uname()[1], round(used_gb, 2), round(total_gb, 2)) +
"The top 5 memory consumers are:\n\n{}".format(proc_str) +
"\n\nIn addition, ~{} GB of shared memory is ".format(
"The top 10 memory consumers are:\n\n{}".format(proc_str) +
"\n\nIn addition, up to {} GB of shared memory is ".format(
round(psutil.virtual_memory().shared / 1e9, 2)) +
"currently being used by the Ray object store. You can set "
"the object store size with the `object_store_memory` "
"parameter when starting Ray, and the max Redis size with "
"`redis_max_memory`.")
"`redis_max_memory`. Note that Ray assumes all system "
"memory is available for use by workers. If your system "
"has other applications running, you should manually set "
"these memory limits to a lower value.")
class MemoryMonitor(object):