Fix O(n^2) behavior in the log_monitor (#5569)

python/ray/log_monitor.py

@@ -8,6 +8,7 @@ import glob
 import json
 import logging
 import os
+import shutil
 import time
 import traceback
 
@@ -85,7 +86,26 @@ class LogMonitor(object):
             file_info = self.open_file_infos.pop(0)
             file_info.file_handle.close()
             file_info.file_handle = None
-            self.closed_file_infos.append(file_info)
+            try:
+                # Test if the worker process that generated the log file
+                # is still alive.
+                os.kill(file_info.worker_pid, 0)
+            except OSError:
+                # The process is not alive any more, so move the log file
+                # out of the log directory so glob.glob will not be slowed
+                # by it.
+                target = os.path.join(self.logs_dir, "old",
+                                      os.path.basename(file_info.filename))
+                try:
+                    shutil.move(file_info.filename, target)
+                except (IOError, OSError) as e:
+                    if e.errno == errno.ENOENT:
+                        logger.warning("Warning: The file {} was not "
+                                       "found.".format(file_info.filename))
+                    else:
+                        raise e
+            else:
+                self.closed_file_infos.append(file_info)
         self.can_open_more_files = True
 
     def update_log_filenames(self):