[autoscaler] Fix filesystem permission race conditions (#8327)

python/ray/autoscaler/gcp/config.py

@@ -1,3 +1,4 @@
+from functools import partial
 import os
 import logging
 import time
@@ -236,9 +237,15 @@ def _configure_key_pair(config):
 
             _create_project_ssh_key_pair(project, public_key, ssh_user)
 
-            with open(private_key_path, "w") as f:
+            # We need to make sure to _create_ the file with the right
+            # permissions. In order to do that we need to change the default
+            # os.open behavior to include the mode we want.
+            with open(
+                    private_key_path,
+                    "w",
+                    opener=partial(os.open, mode=0o600),
+            ) as f:
                 f.write(private_key)
-            os.chmod(private_key_path, 0o600)
 
             with open(public_key_path, "w") as f:
                 f.write(public_key)