wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-10 09:56:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update memory_monitor.py (#9212)

Browse Source
This commit is contained in:
PidgeyBE
2020-08-24 19:29:01 +02:00
committed by GitHub
parent 4761eacc3e
commit a82124d304
1 changed files with 25 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/memory_monitor.py
+25 -20
View File
@@ -102,33 +102,38 @@ class MemoryMonitor:
self.heap_limit = limit_bytes
self.worker_name = worker_name
def get_memory_usage(self):
psutil_mem = psutil.virtual_memory()
total_gb = psutil_mem.total / (1024**3)
used_gb = total_gb - psutil_mem.available / (1024**3)
# Linux, BSD has cached memory, which should
# also be considered as unused memory
if hasattr(psutil_mem, "cached"):
used_gb -= psutil_mem.cached / (1024**3)
if self.cgroup_memory_limit_gb < total_gb:
total_gb = self.cgroup_memory_limit_gb
with open("/sys/fs/cgroup/memory/memory.usage_in_bytes",
"rb") as f:
used_gb = int(f.read()) / (1024**3)
# Exclude the page cache
with open("/sys/fs/cgroup/memory/memory.stat", "r") as f:
for line in f.readlines():
if line.split(" ")[0] == "cache":
used_gb = \
used_gb - int(line.split(" ")[1]) / (1024**3)
assert used_gb >= 0
return used_gb, total_gb
def raise_if_low_memory(self):
if time.time() - self.last_checked > self.check_interval:
if "RAY_DEBUG_DISABLE_MEMORY_MONITOR" in os.environ:
return # escape hatch, not intended for user use
self.last_checked = time.time()
psutil_mem = psutil.virtual_memory()
total_gb = psutil_mem.total / (1024**3)
used_gb = total_gb - psutil_mem.available / (1024**3)
used_gb, total_gb = self.get_memory_usage()
# Linux, BSD has cached memory, which should
# also be considered as unused memory
if hasattr(psutil_mem, "cached"):
used_gb -= psutil_mem.cached / (1024**3)
if self.cgroup_memory_limit_gb < total_gb:
total_gb = self.cgroup_memory_limit_gb
with open("/sys/fs/cgroup/memory/memory.usage_in_bytes",
"rb") as f:
used_gb = int(f.read()) / (1024**3)
# Exclude the page cache
with open("/sys/fs/cgroup/memory/memory.stat", "r") as f:
for line in f.readlines():
if line.split(" ")[0] == "cache":
used_gb = \
used_gb - int(line.split(" ")[1]) / (1024**3)
assert used_gb >= 0
if used_gb > total_gb * self.error_threshold:
raise RayOutOfMemoryError(
RayOutOfMemoryError.get_message(used_gb, total_gb,