[tune] Added Kubernetes syncer and sync client (#10097)

Co-authored-by: Edward Oakes <ed.nmi.oakes@gmail.com>
Co-authored-by: Richard Liaw <rliaw@berkeley.edu>
Co-authored-by: Kai Fricke <kai@anyscale.com>

python/ray/autoscaler/command_runner.py

@@ -215,10 +215,16 @@ class KubernetesCommandRunner(CommandRunnerInterface):
             logger.warning(self.log_prefix +
                            "rsync failed: '{}'. Falling back to 'kubectl cp'"
                            .format(e))
-            self.process_runner.check_call(self.kubectl + [
-                "cp", source, "{}/{}:{}".format(self.namespace, self.node_id,
-                                                target)
-            ])
+            self.run_cp_up(source, target)
+
+    def run_cp_up(self, source, target):
+        if target.startswith("~"):
+            target = "/root" + target[1:]
+
+        self.process_runner.check_call(self.kubectl + [
+            "cp", source, "{}/{}:{}".format(self.namespace, self.node_id,
+                                            target)
+        ])
 
     def run_rsync_down(self, source, target):
         if target.startswith("~"):
@@ -235,10 +241,16 @@ class KubernetesCommandRunner(CommandRunnerInterface):
             logger.warning(self.log_prefix +
                            "rsync failed: '{}'. Falling back to 'kubectl cp'"
                            .format(e))
-            self.process_runner.check_call(self.kubectl + [
-                "cp", "{}/{}:{}".format(self.namespace, self.node_id, source),
-                target
-            ])
+            self.run_cp_down(source, target)
+
+    def run_cp_down(self, source, target):
+        if target.startswith("~"):
+            target = "/root" + target[1:]
+
+        self.process_runner.check_call(self.kubectl + [
+            "cp", "{}/{}:{}".format(self.namespace, self.node_id, source),
+            target
+        ])
 
     def remote_shell_command_str(self):
         return "{} exec -it {} bash".format(" ".join(self.kubectl),

python/ray/autoscaler/kubernetes/kubectl-rsync.sh

@@ -18,7 +18,8 @@ shift
 if [ "X$pod" = "X-l" ]; then
     pod=$1
     shift
-    namespace="-n $1"
+    # Space before $1 leads to namespace errors
+    namespace="-n$1"
     shift
 fi