[tune] fix log dir race condition (#6420)

python/ray/tune/trainable.py

@@ -76,8 +76,8 @@ class Trainable(object):
             self._logdir = self._result_logger.logdir
         else:
             logdir_prefix = datetime.today().strftime("%Y-%m-%d_%H-%M-%S")
-            if not os.path.exists(DEFAULT_RESULTS_DIR):
-                os.makedirs(DEFAULT_RESULTS_DIR)
+            ray.utils.try_to_create_directory(
+                DEFAULT_RESULTS_DIR, warn_if_exist=False)
             self._logdir = tempfile.mkdtemp(
                 prefix=logdir_prefix, dir=DEFAULT_RESULTS_DIR)
             self._result_logger = UnifiedLogger(

python/ray/utils.py

@@ -552,7 +552,6 @@ def try_to_create_directory(directory_path, warn_if_exist=True):
         directory_path: The path of the directory to create.
         warn_if_exist (bool): Warn if the directory already exists.
     """
-    logger = logging.getLogger("ray")
     directory_path = os.path.expanduser(directory_path)
     if not os.path.exists(directory_path):
         try:
@@ -561,6 +560,7 @@ def try_to_create_directory(directory_path, warn_if_exist=True):
             if e.errno != errno.EEXIST:
                 raise e
             if warn_if_exist:
+                logger = logging.getLogger("ray")
                 logger.warning(
                     "Attempted to create '{}', but the directory already "
                     "exists.".format(directory_path))