[docker] Fix rsyncs & filesync for docker (#9920)

python/ray/autoscaler/command_runner.py

@@ -481,14 +481,28 @@ class DockerCommandRunner(SSHCommandRunner):
             ssh_options_override=ssh_options_override)
 
     def run_rsync_up(self, source, target):
+        protected_path = ""
+        if target.find("/root") == 0:
+            protected_path = target
+            target = target.replace("/root", "/tmp/root")
+            self.ssh_command_runner.run(
+                f"mkdir -p {os.path.dirname(target)}", run_env="host")
         self.ssh_command_runner.run_rsync_up(source, target)
         if self._check_container_status():
             self.ssh_command_runner.run("docker cp {} {}:{}".format(
-                target, self.docker_name, self._docker_expand_user(target)))
+                target, self.docker_name,
+                self._docker_expand_user(protected_path)))
 
     def run_rsync_down(self, source, target):
+        protected_path = ""
+        if source.find("/root") == 0:
+            protected_path = source
+            source = source.replace("/root", "/tmp/root")
+            self.ssh_command_runner.run(
+                f"mkdir -p {os.path.dirname(source)}", run_env="host")
         self.ssh_command_runner.run("docker cp {}:{} {}".format(
-            self.docker_name, self._docker_expand_user(source), source))
+            self.docker_name, self._docker_expand_user(protected_path),
+            source))
         self.ssh_command_runner.run_rsync_down(source, target)
 
     def remote_shell_command_str(self):

python/ray/autoscaler/docker.py

@@ -93,8 +93,11 @@ def docker_start_cmds(user, image, mount, cname, user_options):
         "-p {port}:{port}".format(port=port)
         for port in ["6379", "8076", "4321"]
     ])
-    mount_flags = " ".join(
-        ["-v {src}:{dest}".format(src=k, dest=v) for k, v in mount.items()])
+    # TODO(ilr) Move away from defaulting to /root/
+    mount_flags = " ".join([
+        "-v {src}:{dest}".format(src=k, dest=v.replace("~/", "/root/"))
+        for k, v in mount.items()
+    ])
 
     # for click, used in ray cli
     env_vars = {"LC_ALL": "C.UTF-8", "LANG": "C.UTF-8"}