[Object Spilling] Clean up FS storage upon sigint for ray.init(). (#13649)

* Initial iteration done.

* Remove unnecessary messages.

* Addressed code review.

* Addressed code review.

* fix issues.

* addressed code review.

* Addressed the last code review.
This commit is contained in:
SangBin Cho
2021-01-26 23:10:29 -08:00
committed by GitHub
parent 8baafacb1e
commit d2963f4ee1
4 changed files with 146 additions and 40 deletions
+12
View File
@@ -421,6 +421,9 @@ class Node:
"metrics_export_port": self._metrics_export_port
}
def is_head(self):
return self.head
def create_redis_client(self):
"""Create a redis client."""
return ray._private.services.create_redis_client(
@@ -1152,3 +1155,12 @@ class Node:
True if any process that wasn't explicitly killed is still alive.
"""
return not any(self.dead_processes())
def destroy_external_storage(self):
object_spilling_config = self._config.get("object_spilling_config", {})
if object_spilling_config:
object_spilling_config = json.loads(object_spilling_config)
from ray import external_storage
storage = external_storage.setup_external_storage(
object_spilling_config)
storage.destroy_external_storage()