Fix tempfile issues (#4605)

python/ray/node.py

@@ -62,6 +62,7 @@ class Node(object):
         if shutdown_at_exit and connect_only:
             raise ValueError("'shutdown_at_exit' and 'connect_only' cannot "
                              "be both true.")
+        self.head = head
         self.all_processes = {}
 
         # Try to get node IP address with the parameters.
@@ -78,6 +79,7 @@ class Node(object):
             include_log_monitor=True,
             resources={},
             include_webui=False,
+            temp_dir="/tmp/ray",
             worker_path=os.path.join(
                 os.path.dirname(os.path.abspath(__file__)),
                 "workers/default_worker.py"))
@@ -87,7 +89,19 @@ class Node(object):
         self._config = (json.loads(ray_params._internal_config)
                         if ray_params._internal_config else None)
 
-        self._init_temp()
+        if head:
+            redis_client = None
+            # date including microsecond
+            date_str = datetime.datetime.today().strftime(
+                "%Y-%m-%d_%H-%M-%S_%f")
+            self.session_name = "session_{date_str}_{pid}".format(
+                pid=os.getpid(), date_str=date_str)
+        else:
+            redis_client = self.create_redis_client()
+            self.session_name = ray.utils.decode(
+                redis_client.get("session_name"))
+
+        self._init_temp(redis_client)
 
         if connect_only:
             # Get socket names from the configuration.
@@ -119,7 +133,6 @@ class Node(object):
             ray_params.update_if_absent(num_redis_shards=1, include_webui=True)
             self._webui_url = None
         else:
-            redis_client = self.create_redis_client()
             self._webui_url = (
                 ray.services.get_webui_url_from_redis(redis_client))
             ray_params.include_java = (
@@ -128,6 +141,10 @@ class Node(object):
         # Start processes.
         if head:
             self.start_head_processes()
+            redis_client = self.create_redis_client()
+            redis_client.set("session_name", self.session_name)
+            redis_client.set("session_dir", self._session_dir)
+            redis_client.set("temp_dir", self._temp_dir)
 
         if not connect_only:
             self.start_ray_processes()
@@ -136,25 +153,31 @@ class Node(object):
             atexit.register(lambda: self.kill_all_processes(
                 check_alive=False, allow_graceful=True))
 
-    def _init_temp(self):
+    def _init_temp(self, redis_client):
         # Create an dictionary to store temp file index.
         self._incremental_dict = collections.defaultdict(lambda: 0)
 
-        self._temp_dir = self._ray_params.temp_dir
-        if self._temp_dir is None:
-            date_str = datetime.datetime.today().strftime("%Y-%m-%d_%H-%M-%S")
-            self._temp_dir = self._make_inc_temp(
-                prefix="session_{date_str}_{pid}".format(
-                    pid=os.getpid(), date_str=date_str),
-                directory_name="/tmp/ray")
+        if self.head:
+            self._temp_dir = self._ray_params.temp_dir
+        else:
+            self._temp_dir = ray.utils.decode(redis_client.get("temp_dir"))
 
-        try_to_create_directory(self._temp_dir)
+        try_to_create_directory(self._temp_dir, warn_if_exist=False)
+
+        if self.head:
+            self._session_dir = os.path.join(self._temp_dir, self.session_name)
+        else:
+            self._session_dir = ray.utils.decode(
+                redis_client.get("session_dir"))
+
+        # Send a warning message if the session exists.
+        try_to_create_directory(self._session_dir)
         # Create a directory to be used for socket files.
-        self._sockets_dir = os.path.join(self._temp_dir, "sockets")
-        try_to_create_directory(self._sockets_dir)
+        self._sockets_dir = os.path.join(self._session_dir, "sockets")
+        try_to_create_directory(self._sockets_dir, warn_if_exist=False)
         # Create a directory to be used for process log files.
-        self._logs_dir = os.path.join(self._temp_dir, "logs")
-        try_to_create_directory(self._logs_dir)
+        self._logs_dir = os.path.join(self._session_dir, "logs")
+        try_to_create_directory(self._logs_dir, warn_if_exist=False)
 
     @property
     def node_ip_address(self):
@@ -204,6 +227,7 @@ class Node(object):
             "object_store_address": self._plasma_store_socket_name,
             "raylet_socket_name": self._raylet_socket_name,
             "webui_url": self._webui_url,
+            "session_dir": self._session_dir,
         }
 
     def create_redis_client(self):
@@ -215,6 +239,10 @@ class Node(object):
         """Get the path of the temporary directory."""
         return self._temp_dir
 
+    def get_session_dir_path(self):
+        """Get the path of the session directory."""
+        return self._session_dir
+
     def get_logs_dir_path(self):
         """Get the path of the log files directory."""
         return self._logs_dir