diff --git a/python/ray/memory_monitor.py b/python/ray/memory_monitor.py index 40980a09e..f6d6f352c 100644 --- a/python/ray/memory_monitor.py +++ b/python/ray/memory_monitor.py @@ -102,33 +102,38 @@ class MemoryMonitor: self.heap_limit = limit_bytes self.worker_name = worker_name + def get_memory_usage(self): + psutil_mem = psutil.virtual_memory() + total_gb = psutil_mem.total / (1024**3) + used_gb = total_gb - psutil_mem.available / (1024**3) + + # Linux, BSD has cached memory, which should + # also be considered as unused memory + if hasattr(psutil_mem, "cached"): + used_gb -= psutil_mem.cached / (1024**3) + + if self.cgroup_memory_limit_gb < total_gb: + total_gb = self.cgroup_memory_limit_gb + with open("/sys/fs/cgroup/memory/memory.usage_in_bytes", + "rb") as f: + used_gb = int(f.read()) / (1024**3) + # Exclude the page cache + with open("/sys/fs/cgroup/memory/memory.stat", "r") as f: + for line in f.readlines(): + if line.split(" ")[0] == "cache": + used_gb = \ + used_gb - int(line.split(" ")[1]) / (1024**3) + assert used_gb >= 0 + return used_gb, total_gb + def raise_if_low_memory(self): if time.time() - self.last_checked > self.check_interval: if "RAY_DEBUG_DISABLE_MEMORY_MONITOR" in os.environ: return # escape hatch, not intended for user use self.last_checked = time.time() - psutil_mem = psutil.virtual_memory() - total_gb = psutil_mem.total / (1024**3) - used_gb = total_gb - psutil_mem.available / (1024**3) + used_gb, total_gb = self.get_memory_usage() - # Linux, BSD has cached memory, which should - # also be considered as unused memory - if hasattr(psutil_mem, "cached"): - used_gb -= psutil_mem.cached / (1024**3) - - if self.cgroup_memory_limit_gb < total_gb: - total_gb = self.cgroup_memory_limit_gb - with open("/sys/fs/cgroup/memory/memory.usage_in_bytes", - "rb") as f: - used_gb = int(f.read()) / (1024**3) - # Exclude the page cache - with open("/sys/fs/cgroup/memory/memory.stat", "r") as f: - for line in f.readlines(): - if line.split(" ")[0] == "cache": - used_gb = \ - used_gb - int(line.split(" ")[1]) / (1024**3) - assert used_gb >= 0 if used_gb > total_gb * self.error_threshold: raise RayOutOfMemoryError( RayOutOfMemoryError.get_message(used_gb, total_gb,