Increase the number of log file names and handle errors better in log monitor. (#693)

python/ray/log_monitor.py

@@ -3,6 +3,7 @@ from __future__ import division
 from __future__ import print_function
 
 import argparse
+import os
 import redis
 import time
 
@@ -67,8 +68,14 @@ class LogMonitor(object):
       else:
         try:
           self.log_file_handles[log_filename] = open(log_filename, "r")
-        except IOError:
-          print("Warning: The file {} was not found.".format(log_filename))
+        except IOError as e:
+          if e.errno == os.errno.EMFILE:
+            print("Warning: Some files are not being logged because there are "
+                  "too many open files.")
+          elif e.errno == os.errno.ENOENT:
+            print("Warning: The file {} was not found.".format(log_filename))
+          else:
+            raise e
 
   def run(self):
     """Run the log monitor.

python/ray/services.py

@@ -1091,9 +1091,9 @@ def new_log_files(name, redirect_output):
     # Change the log directory permissions so others can use it. This is
     # important when multiple people are using the same machine.
     os.chmod(logs_dir, 0o0777)
-  log_id = random.randint(0, 100000)
-  log_stdout = "{}/{}-{:06d}.out".format(logs_dir, name, log_id)
-  log_stderr = "{}/{}-{:06d}.err".format(logs_dir, name, log_id)
+  log_id = random.randint(0, 1000000000)
+  log_stdout = "{}/{}-{:010d}.out".format(logs_dir, name, log_id)
+  log_stderr = "{}/{}-{:010d}.err".format(logs_dir, name, log_id)
   log_stdout_file = open(log_stdout, "a")
   log_stderr_file = open(log_stderr, "a")
   return log_stdout_file, log_stderr_file