From 73fa94731fed3c56fdb665410661fe2509c7a297 Mon Sep 17 00:00:00 2001 From: Frank Gu Date: Thu, 22 Oct 2020 14:12:51 -0700 Subject: [PATCH] [tune] Add HDFS as Cloud Sync Client (#11524) --- doc/source/tune/user-guide.rst | 2 +- python/ray/tune/sync_client.py | 21 ++++++++++++++++----- python/ray/tune/syncer.py | 10 +++++----- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/doc/source/tune/user-guide.rst b/doc/source/tune/user-guide.rst index 18e9b8b30..7400cc327 100644 --- a/doc/source/tune/user-guide.rst +++ b/doc/source/tune/user-guide.rst @@ -447,7 +447,7 @@ User-provided fields will be outputted automatically on a best-effort basis. You Uploading Results ----------------- -If an upload directory is provided, Tune will automatically sync results from the ``local_dir`` to the given directory, natively supporting standard S3/gsutil URIs. +If an upload directory is provided, Tune will automatically sync results from the ``local_dir`` to the given directory, natively supporting standard S3/gsutil/HDFS URIs. .. code-block:: python diff --git a/python/ray/tune/sync_client.py b/python/ray/tune/sync_client.py index 87f5f4e06..b21a0b06a 100644 --- a/python/ray/tune/sync_client.py +++ b/python/ray/tune/sync_client.py @@ -13,7 +13,8 @@ logger = logging.getLogger(__name__) S3_PREFIX = "s3://" GS_PREFIX = "gs://" -ALLOWED_REMOTE_PREFIXES = (S3_PREFIX, GS_PREFIX) +HDFS_PREFIX = "hdfs://" +ALLOWED_REMOTE_PREFIXES = (S3_PREFIX, GS_PREFIX, HDFS_PREFIX) noop_template = ": {target}" # noop in bash @@ -53,7 +54,7 @@ def get_cloud_sync_client(remote_path): """Returns a CommandBasedClient that can sync to/from remote storage. Args: - remote_path (str): Path to remote storage (S3 or GS). + remote_path (str): Path to remote storage (S3, GS or HDFS). Raises: ValueError if malformed remote_dir. @@ -63,19 +64,29 @@ def get_cloud_sync_client(remote_path): raise ValueError( "Upload uri starting with '{}' requires awscli tool" " to be installed".format(S3_PREFIX)) - template = "aws s3 sync {source} {target} --only-show-errors" + sync_up_template = "aws s3 sync {source} {target} --only-show-errors" + sync_down_template = sync_up_template delete_template = "aws s3 rm {target} --recursive --only-show-errors" elif remote_path.startswith(GS_PREFIX): if not distutils.spawn.find_executable("gsutil"): raise ValueError( "Upload uri starting with '{}' requires gsutil tool" " to be installed".format(GS_PREFIX)) - template = "gsutil rsync -r {source} {target}" + sync_up_template = "gsutil rsync -r {source} {target}" + sync_down_template = sync_up_template delete_template = "gsutil rm -r {target}" + elif remote_path.startswith(HDFS_PREFIX): + if not distutils.spawn.find_executable("hdfs"): + raise ValueError("Upload uri starting with '{}' requires hdfs tool" + " to be installed".format(HDFS_PREFIX)) + sync_up_template = "hdfs dfs -put -f {source} {target}" + sync_down_template = "hdfs dfs -get -f {target} {source}" + delete_template = "hdfs dfs -rm -r {target}" else: raise ValueError("Upload uri must start with one of: {}" "".format(ALLOWED_REMOTE_PREFIXES)) - return CommandBasedClient(template, template, delete_template) + return CommandBasedClient(sync_up_template, sync_down_template, + delete_template) class SyncClient: diff --git a/python/ray/tune/syncer.py b/python/ray/tune/syncer.py index c75f46239..ea74c41de 100644 --- a/python/ray/tune/syncer.py +++ b/python/ray/tune/syncer.py @@ -83,13 +83,13 @@ class SyncConfig: Args: upload_dir (str): Optional URI to sync training results and checkpoints - to (e.g. ``s3://bucket`` or ``gs://bucket``). + to (e.g. ``s3://bucket``, ``gs://bucket`` or ``hdfs://path``). sync_to_cloud (func|str): Function for syncing the local_dir to and from upload_dir. If string, then it must be a string template that includes `{source}` and `{target}` for the syncer to run. If not - provided, the sync command defaults to standard S3 or gsutil sync - commands. By default local_dir is synced to remote_dir every 300 - seconds. To change this, set the TUNE_CLOUD_SYNC_S + provided, the sync command defaults to standard S3, gsutil or HDFS + sync commands. By default local_dir is synced to remote_dir every + 300 seconds. To change this, set the TUNE_CLOUD_SYNC_S environment variable in the driver machine. sync_to_driver (func|str|bool): Function for syncing trial logdir from remote node to local. If string, then it must be a string template @@ -299,7 +299,7 @@ def get_cloud_syncer(local_dir, remote_dir=None, sync_function=None): sync_function (func | str): Function for syncing the local_dir to remote_dir. If string, then it must be a string template for syncer to run. If not provided, it defaults - to standard S3 or gsutil sync commands. + to standard S3, gsutil or HDFS sync commands. Raises: ValueError if malformed remote_dir.