wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-09 14:00:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

[tune] Add HDFS as Cloud Sync Client (#11524)

Browse Source
This commit is contained in:
Frank Gu
2020-10-22 14:12:51 -07:00
committed by GitHub
parent 083737c63c
commit 73fa94731f
3 changed files with 22 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/tune/sync_client.py
+16 -5
View File
@@ -13,7 +13,8 @@ logger = logging.getLogger(__name__)
S3_PREFIX = "s3://"
GS_PREFIX = "gs://"
ALLOWED_REMOTE_PREFIXES = (S3_PREFIX, GS_PREFIX)
HDFS_PREFIX = "hdfs://"
ALLOWED_REMOTE_PREFIXES = (S3_PREFIX, GS_PREFIX, HDFS_PREFIX)
noop_template = ": {target}" # noop in bash
@@ -53,7 +54,7 @@ def get_cloud_sync_client(remote_path):
"""Returns a CommandBasedClient that can sync to/from remote storage.
Args:
remote_path (str): Path to remote storage (S3 or GS).
remote_path (str): Path to remote storage (S3, GS or HDFS).
Raises:
ValueError if malformed remote_dir.
@@ -63,19 +64,29 @@ def get_cloud_sync_client(remote_path):
raise ValueError(
"Upload uri starting with '{}' requires awscli tool"
" to be installed".format(S3_PREFIX))
template = "aws s3 sync {source} {target} --only-show-errors"
sync_up_template = "aws s3 sync {source} {target} --only-show-errors"
sync_down_template = sync_up_template
delete_template = "aws s3 rm {target} --recursive --only-show-errors"
elif remote_path.startswith(GS_PREFIX):
if not distutils.spawn.find_executable("gsutil"):
raise ValueError(
"Upload uri starting with '{}' requires gsutil tool"
" to be installed".format(GS_PREFIX))
template = "gsutil rsync -r {source} {target}"
sync_up_template = "gsutil rsync -r {source} {target}"
sync_down_template = sync_up_template
delete_template = "gsutil rm -r {target}"
elif remote_path.startswith(HDFS_PREFIX):
if not distutils.spawn.find_executable("hdfs"):
raise ValueError("Upload uri starting with '{}' requires hdfs tool"
" to be installed".format(HDFS_PREFIX))
sync_up_template = "hdfs dfs -put -f {source} {target}"
sync_down_template = "hdfs dfs -get -f {target} {source}"
delete_template = "hdfs dfs -rm -r {target}"
else:
raise ValueError("Upload uri must start with one of: {}"
"".format(ALLOWED_REMOTE_PREFIXES))
return CommandBasedClient(template, template, delete_template)
return CommandBasedClient(sync_up_template, sync_down_template,
delete_template)
class SyncClient:
python/ray/tune/syncer.py
+5 -5
View File
@@ -83,13 +83,13 @@ class SyncConfig:
Args:
upload_dir (str): Optional URI to sync training results and checkpoints
to (e.g. ``s3://bucket`` or ``gs://bucket``).
to (e.g. ``s3://bucket``, ``gs://bucket`` or ``hdfs://path``).
sync_to_cloud (func|str): Function for syncing the local_dir to and
from upload_dir. If string, then it must be a string template that
includes `{source}` and `{target}` for the syncer to run. If not
provided, the sync command defaults to standard S3 or gsutil sync
commands. By default local_dir is synced to remote_dir every 300
seconds. To change this, set the TUNE_CLOUD_SYNC_S
provided, the sync command defaults to standard S3, gsutil or HDFS
sync commands. By default local_dir is synced to remote_dir every
300 seconds. To change this, set the TUNE_CLOUD_SYNC_S
environment variable in the driver machine.
sync_to_driver (func|str|bool): Function for syncing trial logdir from
remote node to local. If string, then it must be a string template
@@ -299,7 +299,7 @@ def get_cloud_syncer(local_dir, remote_dir=None, sync_function=None):
sync_function (func | str): Function for syncing the local_dir to
remote_dir. If string, then it must be a string template for
syncer to run. If not provided, it defaults
to standard S3 or gsutil sync commands.
to standard S3, gsutil or HDFS sync commands.
Raises:
ValueError if malformed remote_dir.