From 5b404086784cfbbe8b48a2156984902be344e4ec Mon Sep 17 00:00:00 2001 From: Ujval Misra Date: Fri, 3 Jan 2020 04:03:13 -0500 Subject: [PATCH] [tune] Remove py2.7-specific code (#6665) * Remove backwards compatability py2.7 code. * Use exists_ok=True in ray * nit * nit Co-authored-by: Richard Liaw --- python/ray/node.py | 8 ++++---- python/ray/tune/checkpoint_manager.py | 11 ----------- python/ray/tune/ray_trial_executor.py | 3 +-- python/ray/tune/sync_client.py | 6 ++---- python/ray/tune/syncer.py | 5 +---- python/ray/tune/trainable.py | 9 +++------ python/ray/tune/trial.py | 7 +++---- python/ray/tune/trial_runner.py | 5 ++--- python/ray/utils.py | 16 ++-------------- 9 files changed, 18 insertions(+), 52 deletions(-) diff --git a/python/ray/node.py b/python/ray/node.py index 4b7838730..d3d0dab60 100644 --- a/python/ray/node.py +++ b/python/ray/node.py @@ -198,7 +198,7 @@ class Node: else: self._temp_dir = ray.utils.decode(redis_client.get("temp_dir")) - try_to_create_directory(self._temp_dir, warn_if_exist=False) + try_to_create_directory(self._temp_dir) if self.head: self._session_dir = os.path.join(self._temp_dir, self.session_name) @@ -212,12 +212,12 @@ class Node: try_to_symlink(session_symlink, self._session_dir) # Create a directory to be used for socket files. self._sockets_dir = os.path.join(self._session_dir, "sockets") - try_to_create_directory(self._sockets_dir, warn_if_exist=False) + try_to_create_directory(self._sockets_dir) # Create a directory to be used for process log files. self._logs_dir = os.path.join(self._session_dir, "logs") - try_to_create_directory(self._logs_dir, warn_if_exist=False) + try_to_create_directory(self._logs_dir) old_logs_dir = os.path.join(self._logs_dir, "old") - try_to_create_directory(old_logs_dir, warn_if_exist=False) + try_to_create_directory(old_logs_dir) def get_resource_spec(self): """Resolve and return the current resource spec for the node.""" diff --git a/python/ray/tune/checkpoint_manager.py b/python/ray/tune/checkpoint_manager.py index 36ac41d13..9c745653b 100644 --- a/python/ray/tune/checkpoint_manager.py +++ b/python/ray/tune/checkpoint_manager.py @@ -6,11 +6,6 @@ from __future__ import print_function import heapq import logging -try: - FileNotFoundError -except NameError: - FileNotFoundError = IOError - logger = logging.getLogger(__name__) @@ -44,12 +39,6 @@ class QueueItem: self.priority = priority self.value = value - def __cmp__(self, other): - # For python2.7 compatibility. - if self.priority == other.priority: - return 0 - return -1 if self.priority < other.priority else 1 - def __lt__(self, other): return self.priority < other.priority diff --git a/python/ray/tune/ray_trial_executor.py b/python/ray/tune/ray_trial_executor.py index 67af35f91..7f507ef7d 100644 --- a/python/ray/tune/ray_trial_executor.py +++ b/python/ray/tune/ray_trial_executor.py @@ -111,8 +111,7 @@ class RayTrialExecutor(TrialExecutor): def logger_creator(config): # Set the working dir in the remote process, for user file writes - if not os.path.exists(remote_logdir): - os.makedirs(remote_logdir) + os.makedirs(remote_logdir, exist_ok=True) if not ray.worker._mode() == ray.worker.LOCAL_MODE: os.chdir(remote_logdir) return NoopLogger(config, remote_logdir) diff --git a/python/ray/tune/sync_client.py b/python/ray/tune/sync_client.py index ece09c496..13a88f1c7 100644 --- a/python/ray/tune/sync_client.py +++ b/python/ray/tune/sync_client.py @@ -9,10 +9,8 @@ import subprocess import tempfile import types -try: # py3 - from shlex import quote -except ImportError: # py2 - from pipes import quote +from shlex import quote + from ray.tune.error import TuneError logger = logging.getLogger(__name__) diff --git a/python/ray/tune/syncer.py b/python/ray/tune/syncer.py index 1f2aecc2e..ff7e305b0 100644 --- a/python/ray/tune/syncer.py +++ b/python/ray/tune/syncer.py @@ -7,10 +7,7 @@ import logging import os import time -try: # py3 - from shlex import quote -except ImportError: # py2 - from pipes import quote +from shlex import quote from ray import services from ray.tune.cluster_info import get_ssh_key, get_ssh_user diff --git a/python/ray/tune/trainable.py b/python/ray/tune/trainable.py index 736a23981..9e463c6c1 100644 --- a/python/ray/tune/trainable.py +++ b/python/ray/tune/trainable.py @@ -74,8 +74,7 @@ class TrainableUtil: @staticmethod def make_checkpoint_dir(checkpoint_dir): """Creates a checkpoint directory at the provided path.""" - if not os.path.exists(checkpoint_dir): - os.makedirs(checkpoint_dir) + os.makedirs(checkpoint_dir, exist_ok=True) # Drop marker in directory to identify it as a checkpoint dir. open(os.path.join(checkpoint_dir, ".is_checkpoint"), "a").close() @@ -127,8 +126,7 @@ class Trainable: self._logdir = self._result_logger.logdir else: logdir_prefix = datetime.today().strftime("%Y-%m-%d_%H-%M-%S") - ray.utils.try_to_create_directory( - DEFAULT_RESULTS_DIR, warn_if_exist=False) + ray.utils.try_to_create_directory(DEFAULT_RESULTS_DIR) self._logdir = tempfile.mkdtemp( prefix=logdir_prefix, dir=DEFAULT_RESULTS_DIR) self._result_logger = UnifiedLogger( @@ -411,8 +409,7 @@ class Trainable: path = os.path.join(tmpdir, relpath_name) # This may be a subdirectory, hence not just using tmpdir - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) + os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "wb") as f: f.write(file_contents) diff --git a/python/ray/tune/trial.py b/python/ray/tune/trial.py index 8a692f175..4406fc6c2 100644 --- a/python/ray/tune/trial.py +++ b/python/ray/tune/trial.py @@ -249,8 +249,7 @@ class Trial: @classmethod def create_logdir(cls, identifier, local_dir): local_dir = os.path.expanduser(local_dir) - if not os.path.exists(local_dir): - os.makedirs(local_dir) + os.makedirs(local_dir, exist_ok=True) return tempfile.mkdtemp( prefix="{}_{}".format(identifier[:MAX_LEN_IDENTIFIER], date_str()), dir=local_dir) @@ -260,8 +259,8 @@ class Trial: if not self.result_logger: if not self.logdir: self.logdir = Trial.create_logdir(str(self), self.local_dir) - elif not os.path.exists(self.logdir): - os.makedirs(self.logdir) + else: + os.makedirs(self.logdir, exist_ok=True) self.result_logger = UnifiedLogger( self.config, diff --git a/python/ray/tune/trial_runner.py b/python/ray/tune/trial_runner.py index e90ca6197..cc70c3935 100644 --- a/python/ray/tune/trial_runner.py +++ b/python/ray/tune/trial_runner.py @@ -147,9 +147,8 @@ class TrialRunner: self._stop_queue = [] self._local_checkpoint_dir = local_checkpoint_dir - if self._local_checkpoint_dir and not os.path.exists( - self._local_checkpoint_dir): - os.makedirs(self._local_checkpoint_dir) + if self._local_checkpoint_dir: + os.makedirs(self._local_checkpoint_dir, exist_ok=True) self._remote_checkpoint_dir = remote_checkpoint_dir self._syncer = get_cloud_syncer(local_checkpoint_dir, diff --git a/python/ray/utils.py b/python/ray/utils.py index f64420b1e..60657deda 100644 --- a/python/ray/utils.py +++ b/python/ray/utils.py @@ -545,26 +545,14 @@ def try_make_directory_shared(directory_path): raise -def try_to_create_directory(directory_path, warn_if_exist=True): +def try_to_create_directory(directory_path): """Attempt to create a directory that is globally readable/writable. Args: directory_path: The path of the directory to create. - warn_if_exist (bool): Warn if the directory already exists. """ directory_path = os.path.expanduser(directory_path) - if not os.path.exists(directory_path): - try: - os.makedirs(directory_path) - except OSError as e: - if e.errno != errno.EEXIST: - raise e - if warn_if_exist: - logger = logging.getLogger("ray") - logger.warning( - "Attempted to create '{}', but the directory already " - "exists.".format(directory_path)) - + os.makedirs(directory_path, exist_ok=True) # Change the log directory permissions so others can use it. This is # important when multiple people are using the same machine. try_make_directory_shared(directory_path)