diff --git a/python/ray/autoscaler/aws/config.py b/python/ray/autoscaler/aws/config.py
index 5679d9ae6..91ce9bfb3 100644
--- a/python/ray/autoscaler/aws/config.py
+++ b/python/ray/autoscaler/aws/config.py
@@ -289,6 +289,10 @@ def _configure_key_pair(config):
 
     ec2 = _resource("ec2", config)
 
+    # Writing the new ssh key to the filesystem fails if the ~/.ssh
+    # directory doesn't already exist.
+    os.makedirs(os.path.expanduser("~/.ssh"), exist_ok=True)
+
     # Try a few times to get or create a good key pair.
     MAX_NUM_KEYS = 30
     for i in range(MAX_NUM_KEYS):
diff --git a/python/ray/autoscaler/gcp/config.py b/python/ray/autoscaler/gcp/config.py
index 94fcb5692..4372dcdf3 100644
--- a/python/ray/autoscaler/gcp/config.py
+++ b/python/ray/autoscaler/gcp/config.py
@@ -291,6 +291,10 @@ def _configure_key_pair(config, compute):
                 key_found = True
                 break
 
+        # Writing the new ssh key to the filesystem fails if the ~/.ssh
+        # directory doesn't already exist.
+        os.makedirs(os.path.expanduser("~/.ssh"), exist_ok=True)
+
         # Create a key since it doesn't exist locally or in GCP
         if not key_found and not os.path.exists(private_key_path):
             logger.info("_configure_key_pair: "