Add automatic local GC and plasma debug logs every 10 minutes by default (#12804)

This commit is contained in:
Eric Liang
2020-12-11 17:09:58 -08:00
committed by GitHub
parent abb1eefdc2
commit 1ce745cf44
12 changed files with 89 additions and 20 deletions
+37
View File
@@ -15,6 +15,43 @@ from ray.internal.internal_api import global_gc
logger = logging.getLogger(__name__)
def test_auto_local_gc(shutdown_only):
ray.init(num_cpus=2, _system_config={"local_gc_interval_s": 1})
class ObjectWithCyclicRef:
def __init__(self):
self.loop = self
@ray.remote(num_cpus=1)
class GarbageHolder:
def __init__(self):
gc.disable()
x = ObjectWithCyclicRef()
self.garbage = weakref.ref(x)
def has_garbage(self):
return self.garbage() is not None
try:
gc.disable()
# Local driver.
local_ref = weakref.ref(ObjectWithCyclicRef())
# Remote workers.
actors = [GarbageHolder.remote() for _ in range(2)]
assert local_ref() is not None
assert all(ray.get([a.has_garbage.remote() for a in actors]))
def check_refs_gced():
return (local_ref() is None and
not any(ray.get([a.has_garbage.remote() for a in actors])))
wait_for_condition(check_refs_gced)
finally:
gc.enable()
def test_global_gc(shutdown_only):
cluster = ray.cluster_utils.Cluster()
for _ in range(2):