[tune] logging fixes, better warnings, better cluster support (#3906)

python/ray/tune/cluster_info.py

@@ -12,12 +12,14 @@ def get_ssh_user():
     return getpass.getuser()
 
 
-# TODO(ekl) this currently only works for clusters launched with
-# ray create_or_update
 def get_ssh_key():
-    """Returns ssh key to connecting to cluster workers."""
+    """Returns ssh key to connecting to cluster workers.
 
-    path = os.path.expanduser("~/ray_bootstrap_key.pem")
+    If the env var TUNE_CLUSTER_SSH_KEY is provided, then this key
+    will be used for syncing across different nodes.
+    """
+    path = os.environ.get("TUNE_CLUSTER_SSH_KEY",
+                          os.path.expanduser("~/ray_bootstrap_key.pem"))
     if os.path.exists(path):
         return path
     return None