diff --git a/python/ray/autoscaler/updater.py b/python/ray/autoscaler/updater.py
index cc1899b67..b5bae21ae 100644
--- a/python/ray/autoscaler/updater.py
+++ b/python/ray/autoscaler/updater.py
@@ -172,6 +172,7 @@ class SSHCommandRunner:
             ("ControlMaster", "auto"),
             ("ControlPath", "{}/%C".format(self.ssh_control_path)),
             ("ControlPersist", "10s"),
+            ("IdentitiesOnly", "yes"),
         ]
 
         return ["-i", self.ssh_private_key] + [
@@ -276,8 +277,8 @@ class SSHCommandRunner:
         ])
 
     def remote_shell_command_str(self):
-        return "ssh -i {} {}@{}\n".format(self.ssh_private_key, self.ssh_user,
-                                          self.ssh_ip)
+        return "ssh -o IdentitiesOnly=yes -i {} {}@{}\n".format(
+            self.ssh_private_key, self.ssh_user, self.ssh_ip)
 
 
 class NodeUpdater: