Ray cluster CRD and example CR + multi-ray-cluster operator (#12098)

python/ray/autoscaler/_private/autoscaler.py

@@ -25,7 +25,7 @@ from ray.autoscaler._private.resource_demand_scheduler import \
     get_bin_pack_residual, ResourceDemandScheduler, NodeType, NodeID, NodeIP, \
     ResourceDict
 from ray.autoscaler._private.util import ConcurrentCounter, validate_config, \
-    with_head_node_ip, hash_launch_conf, hash_runtime_conf, \
+    with_head_node_ip, hash_launch_conf, hash_runtime_conf, add_prefix, \
     DEBUG_AUTOSCALING_STATUS, DEBUG_AUTOSCALING_ERROR
 from ray.autoscaler._private.constants import \
     AUTOSCALER_MAX_NUM_FAILURES, AUTOSCALER_MAX_LAUNCH_BATCH, \
@@ -67,8 +67,11 @@ class StandardAutoscaler:
                  max_concurrent_launches=AUTOSCALER_MAX_CONCURRENT_LAUNCHES,
                  max_failures=AUTOSCALER_MAX_NUM_FAILURES,
                  process_runner=subprocess,
-                 update_interval_s=AUTOSCALER_UPDATE_INTERVAL_S):
+                 update_interval_s=AUTOSCALER_UPDATE_INTERVAL_S,
+                 prefix_cluster_info=False):
         self.config_path = config_path
+        # Prefix each line of info string with cluster name if True
+        self.prefix_cluster_info = prefix_cluster_info
         # Keep this before self.reset (self.provider needs to be created
         # exactly once).
         self.provider = None
@@ -685,6 +688,8 @@ class StandardAutoscaler:
             self.load_metrics.get_resource_utilization())
         if _internal_kv_initialized():
             _internal_kv_put(DEBUG_AUTOSCALING_STATUS, tmp, overwrite=True)
+        if self.prefix_cluster_info:
+            tmp = add_prefix(tmp, self.config["cluster_name"])
         logger.debug(tmp)
 
     def info_string(self, nodes):

python/ray/autoscaler/_private/kubernetes/__init__.py

@@ -5,6 +5,7 @@ _configured = False
 _core_api = None
 _auth_api = None
 _extensions_beta_api = None
+_custom_objects_api = None
 
 
 def _load_config():
@@ -45,4 +46,13 @@ def extensions_beta_api():
     return _extensions_beta_api
 
 
+def custom_objects_api():
+    global _custom_objects_api
+    if _custom_objects_api is None:
+        _load_config()
+        _custom_objects_api = kubernetes.client.CustomObjectsApi()
+
+    return _custom_objects_api
+
+
 log_prefix = "KubernetesNodeProvider: "

python/ray/autoscaler/_private/kubernetes/config.py

@@ -1,4 +1,6 @@
+import copy
 import logging
+import math
 
 from kubernetes import client
 from kubernetes.client.rest import ApiException
@@ -45,9 +47,10 @@ def not_provided_msg(resource_type):
 
 def bootstrap_kubernetes(config):
     if not config["provider"]["use_internal_ips"]:
-        return ValueError("Exposing external IP addresses for ray pods isn't "
-                          "currently supported. Please set "
-                          "'use_internal_ips' to false.")
+        return ValueError(
+            "Exposing external IP addresses for ray containers isn't "
+            "currently supported. Please set "
+            "'use_internal_ips' to false.")
     namespace = _configure_namespace(config["provider"])
     _configure_autoscaler_service_account(namespace, config["provider"])
     _configure_autoscaler_role(namespace, config["provider"])
@@ -56,6 +59,62 @@ def bootstrap_kubernetes(config):
     return config
 
 
+def fillout_resources_kubernetes(config):
+    if "available_node_types" not in config:
+        return config["available_node_types"]
+    node_types = copy.deepcopy(config["available_node_types"])
+    for node_type in node_types:
+        container_data = node_types[node_type]["node_config"]["spec"][
+            "containers"][0]
+        autodetected_resources = get_autodetected_resources(container_data)
+        if "resources" not in config["available_node_types"][node_type]:
+            config["available_node_types"][node_type]["resources"] = {}
+        config["available_node_types"][node_type]["resources"].update(
+            autodetected_resources)
+        logger.debug(
+            "Updating the resources of node type {} to include {}.".format(
+                node_type, autodetected_resources))
+    return config
+
+
+def get_autodetected_resources(container_data):
+    container_resources = container_data.get("resources", None)
+    if container_resources is None:
+        return {"CPU": 0, "GPU": 0}
+
+    node_type_resources = {
+        resource_name.upper(): get_resource(container_resources, resource_name)
+        for resource_name in ["cpu", "gpu"]
+    }
+
+    return node_type_resources
+
+
+def get_resource(container_resources, resource_name):
+    request = _get_resource(
+        container_resources, resource_name, field_name="requests")
+    limit = _get_resource(
+        container_resources, resource_name, field_name="limits")
+    resource = min(request, limit)
+    return 0 if resource == float("inf") else int(resource)
+
+
+def _get_resource(container_resources, resource_name, field_name):
+    if (field_name in container_resources
+            and resource_name in container_resources[field_name]):
+        return _parse_resource(container_resources[field_name][resource_name])
+    else:
+        return float("inf")
+
+
+def _parse_resource(resource):
+    resource_str = str(resource)
+    if resource_str[-1] == "m":
+        return math.ceil(int(resource_str[:-1]) / 1000)
+    else:
+        return int(resource_str)
+
+
 def _configure_namespace(provider_config):
     namespace_field = "namespace"
     if namespace_field not in provider_config:

python/ray/autoscaler/_private/kubernetes/node_provider.py

@@ -6,7 +6,8 @@ from kubernetes.client.rest import ApiException
 from ray.autoscaler._private.command_runner import KubernetesCommandRunner
 from ray.autoscaler._private.kubernetes import core_api, log_prefix, \
     extensions_beta_api
-from ray.autoscaler._private.kubernetes.config import bootstrap_kubernetes
+from ray.autoscaler._private.kubernetes.config import bootstrap_kubernetes, \
+    fillout_resources_kubernetes
 from ray.autoscaler.node_provider import NodeProvider
 from ray.autoscaler.tags import TAG_RAY_CLUSTER_NAME
 
@@ -177,6 +178,11 @@ class KubernetesNodeProvider(NodeProvider):
     def bootstrap_config(cluster_config):
         return bootstrap_kubernetes(cluster_config)
 
+    @staticmethod
+    def fillout_available_node_types_resources(cluster_config):
+        """Fills out missing "resources" field for available_node_types."""
+        return fillout_resources_kubernetes(cluster_config)
+
 
 def _add_service_name_to_service_port(spec, svc_name):
     """Goes recursively through the ingress manifest and adds the

python/ray/autoscaler/_private/util.py

@@ -244,3 +244,14 @@ def hash_runtime_conf(file_mounts,
         file_mounts_contents_hash = None
 
     return (_hash_cache[conf_str], file_mounts_contents_hash)
+
+
+def add_prefix(info_string, prefix):
+    """Prefixes each line of info_string, except the first, by prefix."""
+    lines = info_string.split("\n")
+    prefixed_lines = [lines[0]]
+    for line in lines[1:]:
+        prefixed_line = ":".join([prefix, line])
+        prefixed_lines.append(prefixed_line)
+    prefixed_info_string = "\n".join(prefixed_lines)
+    return prefixed_info_string

python/ray/autoscaler/kubernetes/operator_configs/example_cluster.yaml

@@ -0,0 +1,128 @@
+apiVersion: cluster.ray.io/v1
+kind: RayCluster
+metadata:
+  name: example-cluster
+spec:
+  # The maximum number of workers nodes to launch in addition to the head node.
+  maxWorkers: 3
+  # The autoscaler will scale up the cluster faster with higher upscaling speed.
+  # E.g., if the task requires adding more nodes then autoscaler will gradually
+  # scale up the cluster in chunks of upscaling_speed*currently_running_nodes.
+  # This number should be > 0.
+  upscalingSpeed: 1.0
+  # If a node is idle for this many minutes, it will be removed.
+  idleTimeoutMinutes: 5
+  # Specify the pod type for the ray head node (as configured below).
+  headPodType: head-node
+  # Specify the default pod type for ray the worker nodes (as configured below).
+  workerDefaultPodType: worker-nodes
+  # Specify the allowed pod types for this ray cluster and the resources they provide.
+  podTypes:
+  - name: head-node
+    podConfig:
+      apiVersion: v1
+      kind: Pod
+      metadata:
+        # Automatically generates a name for the pod with this prefix.
+        generateName: example-cluster-ray-head-
+      spec:
+        restartPolicy: Never
+
+        # This volume allocates shared memory for Ray to use for its plasma
+        # object store. If you do not provide this, Ray will fall back to
+        # /tmp which cause slowdowns if is not a shared memory volume.
+        volumes:
+        - name: dshm
+          emptyDir:
+            medium: Memory
+        containers:
+        - name: ray-node
+          imagePullPolicy: Always
+          image: rayproject/ray:nightly
+          # Do not change this command - it keeps the pod alive until it is
+          # explicitly killed.
+          command: ["/bin/bash", "-c", "--"]
+          args: ['trap : TERM INT; sleep infinity & wait;']
+          ports:
+          - containerPort: 6379 # Redis port.
+          - containerPort: 12345 # Ray internal communication.
+          - containerPort: 12346 # Ray internal communication.
+
+          # This volume allocates shared memory for Ray to use for its plasma
+          # object store. If you do not provide this, Ray will fall back to
+          # /tmp which cause slowdowns if is not a shared memory volume.
+          volumeMounts:
+          - mountPath: /dev/shm
+            name: dshm
+          resources:
+            requests:
+              cpu: 1000m
+              memory: 512Mi
+            limits:
+              # The maximum memory that this pod is allowed to use. The
+              # limit will be detected by ray and split to use 10% for
+              # redis, 30% for the shared memory object store, and the
+              # rest for application memory. If this limit is not set and
+              # the object store size is not set manually, ray will
+              # allocate a very large object store in each pod that may
+              # cause problems for other pods.
+              memory: 512Mi
+  - name: worker-nodes
+    # Minimum number of Ray workers of this Pod type.
+    minWorkers: 2
+    # Maximum number of Ray workers of this Pod type. Takes precedence over minWorkers.
+    maxWorkers: 3
+    # User-specified custom resources for use by Ray 
+    rayResources: {"Custom1": 1, "is_spot": 1}
+    # Optional commands to run before starting the Ray runtime.
+    setupCommands: 
+      - pip install numpy # Example
+    podConfig:
+      apiVersion: v1
+      kind: Pod
+      metadata:
+        # Automatically generates a name for the pod with this prefix.
+        generateName: example-cluster-ray-worker-
+      spec:
+        restartPolicy: Never
+        volumes:
+        - name: dshm
+          emptyDir:
+            medium: Memory
+        containers:
+        - name: ray-node
+          imagePullPolicy: Always
+          image: rayproject/ray:nightly
+          command: ["/bin/bash", "-c", "--"]
+          args: ["trap : TERM INT; sleep infinity & wait;"]
+          ports:
+          - containerPort: 12345 # Ray internal communication.
+          - containerPort: 12346 # Ray internal communication.
+          # This volume allocates shared memory for Ray to use for its plasma
+          # object store. If you do not provide this, Ray will fall back to
+          # /tmp which cause slowdowns if is not a shared memory volume.
+          volumeMounts:
+          - mountPath: /dev/shm
+            name: dshm
+          resources:
+            requests:
+              cpu: 1000m
+              memory: 512Mi
+            limits:
+              # The maximum memory that this pod is allowed to use. The
+              # limit will be detected by ray and split to use 10% for
+              # redis, 30% for the shared memory object store, and the
+              # rest for application memory. If this limit is not set and
+              # the object store size is not set manually, ray will
+              # allocate a very large object store in each pod that may
+              # cause problems for other pods.
+              memory: 512Mi
+  # Commands to start Ray on the head node. You don't need to change this.
+  # Note dashboard-host is set to 0.0.0.0 so that Kubernetes can port forward.
+  headStartRayCommands:
+      - ray stop
+      - ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076  --dashboard-host 0.0.0.0
+  # Commands to start Ray on worker nodes. You don't need to change this.
+  workerStartRayCommands:
+      - ray stop
+      - ulimit -n 65536; ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076

python/ray/autoscaler/kubernetes/operator_configs/example_cluster2.yaml

@@ -0,0 +1,128 @@
+apiVersion: cluster.ray.io/v1
+kind: RayCluster
+metadata:
+  name: example-cluster2
+spec:
+  # The maximum number of workers nodes to launch in addition to the head node.
+  maxWorkers: 3
+  # The autoscaler will scale up the cluster faster with higher upscaling speed.
+  # E.g., if the task requires adding more nodes then autoscaler will gradually
+  # scale up the cluster in chunks of upscaling_speed*currently_running_nodes.
+  # This number should be > 0.
+  upscalingSpeed: 1.0
+  # If a node is idle for this many minutes, it will be removed.
+  idleTimeoutMinutes: 5
+  # Specify the pod type for the ray head node (as configured below).
+  headPodType: head-node
+  # Specify the default pod type for ray the worker nodes (as configured below).
+  workerDefaultPodType: worker-nodes
+  # Specify the allowed pod types for this ray cluster and the resources they provide.
+  podTypes:
+  - name: head-node
+    podConfig:
+      apiVersion: v1
+      kind: Pod
+      metadata:
+        # Automatically generates a name for the pod with this prefix.
+        generateName: example-cluster2-ray-head-
+      spec:
+        restartPolicy: Never
+
+        # This volume allocates shared memory for Ray to use for its plasma
+        # object store. If you do not provide this, Ray will fall back to
+        # /tmp which cause slowdowns if is not a shared memory volume.
+        volumes:
+        - name: dshm
+          emptyDir:
+            medium: Memory
+        containers:
+        - name: ray-node
+          imagePullPolicy: Always
+          image: rayproject/ray:nightly
+          # Do not change this command - it keeps the pod alive until it is
+          # explicitly killed.
+          command: ["/bin/bash", "-c", "--"]
+          args: ['trap : TERM INT; sleep infinity & wait;']
+          ports:
+          - containerPort: 6379 # Redis port.
+          - containerPort: 12345 # Ray internal communication.
+          - containerPort: 12346 # Ray internal communication.
+
+          # This volume allocates shared memory for Ray to use for its plasma
+          # object store. If you do not provide this, Ray will fall back to
+          # /tmp which cause slowdowns if is not a shared memory volume.
+          volumeMounts:
+          - mountPath: /dev/shm
+            name: dshm
+          resources:
+            requests:
+              cpu: 1000m
+              memory: 512Mi
+            limits:
+              # The maximum memory that this pod is allowed to use. The
+              # limit will be detected by ray and split to use 10% for
+              # redis, 30% for the shared memory object store, and the
+              # rest for application memory. If this limit is not set and
+              # the object store size is not set manually, ray will
+              # allocate a very large object store in each pod that may
+              # cause problems for other pods.
+              memory: 512Mi
+  - name: worker-nodes
+    # Minimum number of Ray workers of this Pod type.
+    minWorkers: 1
+    # Maximum number of Ray workers of this Pod type. Takes precedence over minWorkers.
+    maxWorkers: 3
+    # User-specified custom resources for use by Ray 
+    rayResources: {"Custom1": 1, "is_spot": 1}
+    # Optional commands to run before starting the Ray runtime.
+    setupCommands: 
+      - pip install numpy # Example
+    podConfig:
+      apiVersion: v1
+      kind: Pod
+      metadata:
+        # Automatically generates a name for the pod with this prefix.
+        generateName: example-cluster2-ray-worker-
+      spec:
+        restartPolicy: Never
+        volumes:
+        - name: dshm
+          emptyDir:
+            medium: Memory
+        containers:
+        - name: ray-node
+          imagePullPolicy: Always
+          image: rayproject/ray:nightly
+          command: ["/bin/bash", "-c", "--"]
+          args: ["trap : TERM INT; sleep infinity & wait;"]
+          ports:
+          - containerPort: 12345 # Ray internal communication.
+          - containerPort: 12346 # Ray internal communication.
+          # This volume allocates shared memory for Ray to use for its plasma
+          # object store. If you do not provide this, Ray will fall back to
+          # /tmp which cause slowdowns if is not a shared memory volume.
+          volumeMounts:
+          - mountPath: /dev/shm
+            name: dshm
+          resources:
+            requests:
+              cpu: 1000m
+              memory: 512Mi
+            limits:
+              # The maximum memory that this pod is allowed to use. The
+              # limit will be detected by ray and split to use 10% for
+              # redis, 30% for the shared memory object store, and the
+              # rest for application memory. If this limit is not set and
+              # the object store size is not set manually, ray will
+              # allocate a very large object store in each pod that may
+              # cause problems for other pods.
+              memory: 512Mi
+  # Commands to start Ray on the head node. You don't need to change this.
+  # Note dashboard-host is set to 0.0.0.0 so that Kubernetes can port forward.
+  headStartRayCommands:
+      - ray stop
+      - ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076  --dashboard-host 0.0.0.0
+  # Commands to start Ray on worker nodes. You don't need to change this.
+  workerStartRayCommands:
+      - ray stop
+      - ulimit -n 65536; ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076

python/ray/autoscaler/kubernetes/operator_configs/operator.yaml

@@ -9,8 +9,8 @@ apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: ray-operator-role
 rules:
-- apiGroups: ["", "rbac.authorization.k8s.io"]
-  resources: ["configmaps", "pods", "pods/exec", "services", "serviceaccounts", "roles", "rolebindings"]
+- apiGroups: ["", "cluster.ray.io"]
+  resources: ["rayclusters", "pods", "pods/exec"]
   verbs: ["get", "watch", "list", "create", "delete", "patch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -35,8 +35,7 @@ spec:
   - name: ray
     imagePullPolicy: Always
     image: rayproject/ray:nightly
-    command: ["/bin/bash", "-c", "--"]
-    args: ["ray-operator; trap : TERM INT; sleep infinity & wait;"]
+    command: ["ray-operator"]
     env:
     - name: RAY_OPERATOR_POD_NAMESPACE
       valueFrom:

python/ray/autoscaler/kubernetes/operator_configs/test_cluster_config.yaml

@@ -1,260 +0,0 @@
-# An unique identifier for the head node and workers of this cluster.
-cluster_name: default
-
-# The autoscaler will scale up the cluster to this target fraction of resource
-# usage. For example, if a cluster of 10 nodes is 100% busy and
-# target_utilization is 0.8, it would resize the cluster to 13. This fraction
-# can be decreased to increase the aggressiveness of upscaling.
-# This value must be less than 1.0 for scaling to happen.
-target_utilization_fraction: 0.8
-
-# If a node is idle for this many minutes, it will be removed.
-idle_timeout_minutes: 5
-
-# Kubernetes resources that need to be configured for the autoscaler to be
-# able to manage the Ray cluster. If any of the provided resources don't
-# exist, the autoscaler will attempt to create them. If this fails, you may
-# not have the required permissions and will have to request them to be
-# created by your cluster administrator.
-provider:
-    type: kubernetes
-
-    # Exposing external IP addresses for ray pods isn't currently supported.
-    use_internal_ips: true
-
-    # Namespace to use for all resources created.
-    namespace: ray
-
-    services:
-      # Service that maps to the head node of the Ray cluster.
-      - apiVersion: v1
-        kind: Service
-        metadata:
-            # NOTE: If you're running multiple Ray clusters with services
-            # on one Kubernetes cluster, they must have unique service
-            # names.
-            name: ray-head
-        spec:
-            # This selector must match the head node pod's selector below.
-            selector:
-                component: ray-head
-            ports:
-                - protocol: TCP
-                  port: 8000
-                  targetPort: 8000
-
-      # Service that maps to the worker nodes of the Ray cluster.
-      - apiVersion: v1
-        kind: Service
-        metadata:
-            # NOTE: If you're running multiple Ray clusters with services
-            # on one Kubernetes cluster, they must have unique service
-            # names.
-            name: ray-workers
-        spec:
-            # This selector must match the worker node pods' selector below.
-            selector:
-                component: ray-worker
-            ports:
-                - protocol: TCP
-                  port: 8000
-                  targetPort: 8000
-
-# Kubernetes pod config for the head node pod.
-available_node_types:
-  head_node:
-      resources: {}
-      node_config:
-        apiVersion: v1
-        kind: Pod
-        metadata:
-            # Automatically generates a name for the pod with this prefix.
-            generateName: ray-head-
-
-            # Must match the head node service selector above if a head node
-            # service is required.
-            labels:
-                component: ray-head
-        spec:
-            # Restarting the head node automatically is not currently supported.
-            # If the head node goes down, `ray up` must be run again.
-            restartPolicy: Never
-
-            # This volume allocates shared memory for Ray to use for its plasma
-            # object store. If you do not provide this, Ray will fall back to
-            # /tmp which cause slowdowns if is not a shared memory volume.
-            volumes:
-            - name: dshm
-              emptyDir:
-                  medium: Memory
-
-            containers:
-            - name: ray-node
-              imagePullPolicy: Always
-              # You are free (and encouraged) to use your own container image,
-              # but it should have the following installed:
-              #   - rsync (used for `ray rsync` commands and file mounts)
-              #   - screen (used for `ray attach`)
-              #   - kubectl (used by the autoscaler to manage worker pods)
-              image: rayproject/ray:nightly
-              # Do not change this command - it keeps the pod alive until it is
-              # explicitly killed.
-              command: ["/bin/bash", "-c", "--"]
-              args: ["trap : TERM INT; sleep infinity & wait;"]
-              ports:
-                  - containerPort: 6379 # Redis port.
-                  - containerPort: 6380 # Redis port.
-                  - containerPort: 6381 # Redis port.
-                  - containerPort: 12345 # Ray internal communication.
-                  - containerPort: 12346 # Ray internal communication.
-
-              # This volume allocates shared memory for Ray to use for its plasma
-              # object store. If you do not provide this, Ray will fall back to
-              # /tmp which cause slowdowns if is not a shared memory volume.
-              volumeMounts:
-                  - mountPath: /dev/shm
-                    name: dshm
-              resources:
-                  requests:
-                      cpu: 1000m
-                      memory: 512Mi
-                  limits:
-                      # The maximum memory that this pod is allowed to use. The
-                      # limit will be detected by ray and split to use 10% for
-                      # redis, 30% for the shared memory object store, and the
-                      # rest for application memory. If this limit is not set and
-                      # the object store size is not set manually, ray will
-                      # allocate a very large object store in each pod that may
-                      # cause problems for other pods.
-                      memory: 2Gi
-              env:
-                  # This is used in the head_start_ray_commands below so that
-                  # Ray can spawn the correct number of processes. Omitting this
-                  # may lead to degraded performance.
-                  - name: MY_CPU_REQUEST
-                    valueFrom:
-                        resourceFieldRef:
-                            resource: requests.cpu
-
-  worker_nodes:
-      resources: {}
-      min_workers: 1
-      max_workers: 2
-      node_config:
-        apiVersion: v1
-        kind: Pod
-        metadata:
-            # Automatically generates a name for the pod with this prefix.
-            generateName: ray-worker-
-
-            # Must match the worker node service selector above if a worker node
-            # service is required.
-            labels:
-                component: ray-worker
-        spec:
-            serviceAccountName: default
-
-            # Worker nodes will be managed automatically by the head node, so
-            # do not change the restart policy.
-            restartPolicy: Never
-
-            # This volume allocates shared memory for Ray to use for its plasma
-            # object store. If you do not provide this, Ray will fall back to
-            # /tmp which cause slowdowns if is not a shared memory volume.
-            volumes:
-            - name: dshm
-              emptyDir:
-                  medium: Memory
-
-            containers:
-            - name: ray-node
-              imagePullPolicy: Always
-              # You are free (and encouraged) to use your own container image,
-              # but it should have the following installed:
-              #   - rsync (used for `ray rsync` commands and file mounts)
-              image: rayproject/ray:nightly
-              # Do not change this command - it keeps the pod alive until it is
-              # explicitly killed.
-              command: ["/bin/bash", "-c", "--"]
-              args: ["trap : TERM INT; sleep infinity & wait;"]
-              ports:
-                  - containerPort: 12345 # Ray internal communication.
-                  - containerPort: 12346 # Ray internal communication.
-
-              # This volume allocates shared memory for Ray to use for its plasma
-              # object store. If you do not provide this, Ray will fall back to
-              # /tmp which cause slowdowns if is not a shared memory volume.
-              volumeMounts:
-                  - mountPath: /dev/shm
-                    name: dshm
-              resources:
-                  requests:
-                      cpu: 100m
-                      memory: 512Mi
-                  limits:
-                      # This memory limit will be detected by ray and split into
-                      # 30% for plasma, and 70% for workers.
-                      memory: 2Gi
-              env:
-                  # This is used in the head_start_ray_commands below so that
-                  # Ray can spawn the correct number of processes. Omitting this
-                  # may lead to degraded performance.
-                  - name: MY_CPU_REQUEST
-                    valueFrom:
-                        resourceFieldRef:
-                            resource: requests.cpu
-
-head_node_type:
-  head_node
-
-worker_default_node_type:
-  worker_nodes
-# Files or directories to copy to the head and worker nodes. The format is a
-# dictionary from REMOTE_PATH: LOCAL_PATH, e.g.
-file_mounts: {
-}
-
-# Files or directories to copy from the head node to the worker nodes. The format is a
-# list of paths. The same path on the head node will be copied to the worker node.
-# This behavior is a subset of the file_mounts behavior. In the vast majority of cases
-# you should just use file_mounts. Only use this if you know what you're doing!
-cluster_synced_files: []
-
-# Whether changes to directories in file_mounts or cluster_synced_files in the head node
-# should sync to the worker node continuously
-file_mounts_sync_continuously: False
-
-# Patterns for files to exclude when running rsync up or rsync down.
-# This is not supported on kubernetes.
-rsync_exclude: []
-
-# Pattern files to use for filtering out files when running rsync up or rsync down. The file is searched for
-# in the source directory and recursively through all subdirectories. For example, if .gitignore is provided
-# as a value, the behavior will match git's behavior for finding and using .gitignore files.
-# This is not supported on kubernetes.
-rsync_filter: []
-
-# List of commands that will be run before `setup_commands`. If docker is
-# enabled, these commands will run outside the container and before docker
-# is setup.
-initialization_commands: []
-
-# List of shell commands to run to set up nodes.
-setup_commands: []
-
-# Custom commands that will be run on the head node after common setup.
-head_setup_commands: []
-
-# Custom commands that will be run on worker nodes after common setup.
-worker_setup_commands: []
-
-# Command to start ray on the head node. You don't need to change this.
-# Note webui-host is set to 0.0.0.0 so that kubernetes can port forward.
-head_start_ray_commands:
-    - ray stop
-    - ulimit -n 65536; ray start --head --num-cpus=$MY_CPU_REQUEST --object-manager-port=8076 --dashboard-host 0.0.0.0
-
-# Command to start ray on worker nodes. You don't need to change this.
-worker_start_ray_commands:
-    - ray stop
-    - ulimit -n 65536; ray start --num-cpus=$MY_CPU_REQUEST --address=$RAY_HEAD_IP:6379 --object-manager-port=8076

python/ray/autoscaler/ray-schema.json

@@ -20,7 +20,7 @@
     "additionalProperties": false,
 	"properties": {
         "cluster_name": {
-            "description": "An unique identifier for the head node and workers of this cluster.",
+            "description": "A unique identifier for the head node and workers of this cluster.",
             "type": "string"
         },
         "min_workers": {

python/ray/monitor.py

@@ -85,7 +85,11 @@ class Monitor:
             This is used to receive notifications about failed components.
     """
 
-    def __init__(self, redis_address, autoscaling_config, redis_password=None):
+    def __init__(self,
+                 redis_address,
+                 autoscaling_config,
+                 redis_password=None,
+                 prefix_cluster_info=False):
         # Initialize the Redis clients.
         ray.state.state._initialize_global_state(
             redis_address, redis_password=redis_password)
@@ -107,8 +111,10 @@ class Monitor:
         head_node_ip = redis_address.split(":")[0]
         self.load_metrics = LoadMetrics(local_ip=head_node_ip)
         if autoscaling_config:
-            self.autoscaler = StandardAutoscaler(autoscaling_config,
-                                                 self.load_metrics)
+            self.autoscaler = StandardAutoscaler(
+                autoscaling_config,
+                self.load_metrics,
+                prefix_cluster_info=prefix_cluster_info)
             self.autoscaling_config = autoscaling_config
         else:
             self.autoscaler = None

python/ray/operator.py

@@ -1,108 +0,0 @@
-"""
-Ray operator for Kubernetes.
-
-Reads ray cluster config from a k8s ConfigMap, starts a ray head node pod using
-create_or_update_cluster(), then runs an autoscaling loop in the operator pod
-executing this script. Writes autoscaling logs to the directory
-/root/ray-operator-logs.
-
-In this setup, the ray head node does not run an autoscaler. It is important
-NOT to supply an --autoscaling-config argument to head node's ray start command
-in the cluster config when using this operator.
-
-To run, first create a ConfigMap named ray-operator-configmap from a ray
-cluster config. Then apply the manifest at python/ray/autoscaler/kubernetes/operator_configs/operator_config.yaml
-
-For example:
-kubectl create namespace raytest
-kubectl -n raytest create configmap ray-operator-configmap --from-file=python/ray/autoscaler/kubernetes/operator_configs/test_cluster_config.yaml
-kubectl -n raytest apply -f python/ray/autoscaler/kubernetes/operator_configs/operator_config.yaml
-""" # noqa
-import os
-from typing import Any, Dict, IO, Tuple
-
-import kubernetes
-import yaml
-
-from ray._private import services
-from ray.autoscaler._private.commands import create_or_update_cluster
-from ray.autoscaler._private.kubernetes import core_api
-from ray.utils import open_log
-from ray import ray_constants
-
-RAY_CLUSTER_NAMESPACE = os.environ.get("RAY_OPERATOR_POD_NAMESPACE")
-RAY_CONFIG_MAP = "ray-operator-configmap"
-RAY_CONFIG_DIR = "/root"
-
-LOG_DIR = "/root/ray-operator-logs"
-ERR_NAME, OUT_NAME = "ray-operator.err", "ray-operator.out"
-
-
-def prepare_ray_cluster_config() -> str:
-    config_map = core_api().read_namespaced_config_map(
-        name=RAY_CONFIG_MAP, namespace=RAY_CLUSTER_NAMESPACE)
-
-    # config_map.data consists of a single key:value pair
-    for config_file_name, config_string in config_map.data.items():
-        config = yaml.safe_load(config_string)
-        config["provider"]["namespace"] = RAY_CLUSTER_NAMESPACE
-        cluster_config_path = os.path.join(RAY_CONFIG_DIR, config_file_name)
-        with open(cluster_config_path, "w") as file:
-            yaml.dump(config, file)
-
-    return cluster_config_path
-
-
-def get_ray_head_pod_ip(config: Dict[str, Any]) -> str:
-    cluster_name = config["cluster_name"]
-    label_selector = f"component=ray-head,ray-cluster-name={cluster_name}"
-    pods = core_api().list_namespaced_pod(
-        namespace=RAY_CLUSTER_NAMESPACE, label_selector=label_selector).items
-    assert (len(pods)) == 1
-    head_pod = pods.pop()
-    return head_pod.status.pod_ip
-
-
-def get_logs() -> Tuple[IO, IO]:
-    try:
-        os.makedirs(LOG_DIR)
-    except OSError:
-        pass
-
-    err_path = os.path.join(LOG_DIR, ERR_NAME)
-    out_path = os.path.join(LOG_DIR, OUT_NAME)
-
-    return open_log(err_path), open_log(out_path)
-
-
-def main():
-    kubernetes.config.load_incluster_config()
-    cluster_config_path = prepare_ray_cluster_config()
-
-    config = create_or_update_cluster(
-        cluster_config_path,
-        override_min_workers=None,
-        override_max_workers=None,
-        no_restart=False,
-        restart_only=False,
-        yes=True,
-        no_config_cache=True)
-    with open(cluster_config_path, "w") as file:
-        yaml.dump(config, file)
-
-    ray_head_pod_ip = get_ray_head_pod_ip(config)
-    # TODO: Add support for user-specified redis port and password
-    redis_address = services.address(ray_head_pod_ip,
-                                     ray_constants.DEFAULT_PORT)
-    stderr_file, stdout_file = get_logs()
-
-    services.start_monitor(
-        redis_address,
-        stdout_file=stdout_file,
-        stderr_file=stderr_file,
-        autoscaling_config=cluster_config_path,
-        redis_password=ray_constants.REDIS_DEFAULT_PASSWORD)
-
-
-if __name__ == "__main__":
-    main()

python/ray/operator/operator.py

@@ -0,0 +1,154 @@
+"""
+Ray operator for Kubernetes.
+
+Reads ray cluster config from a k8s ConfigMap, starts a ray head node pod using
+create_or_update_cluster(), then runs an autoscaling loop in the operator pod
+executing this script. Writes autoscaling logs to the directory
+/root/ray-operator-logs.
+
+In this setup, the ray head node does not run an autoscaler. It is important
+NOT to supply an --autoscaling-config argument to head node's ray start command
+in the cluster config when using this operator.
+
+To run, first create a ConfigMap named ray-operator-configmap from a ray
+cluster config. Then apply the manifest at python/ray/autoscaler/kubernetes/operator_configs/operator_config.yaml
+
+For example:
+kubectl create namespace raytest
+kubectl -n raytest create configmap ray-operator-configmap --from-file=python/ray/autoscaler/kubernetes/operator_configs/test_cluster_config.yaml
+kubectl -n raytest apply -f python/ray/autoscaler/kubernetes/operator_configs/operator_config.yaml
+""" # noqa
+import logging
+import multiprocessing as mp
+import os
+from typing import Any, Callable, Dict, Optional
+
+from kubernetes.client.exceptions import ApiException
+import yaml
+
+from ray._private import services
+from ray.autoscaler._private import commands
+from ray import monitor
+from ray.operator import operator_utils
+from ray import ray_constants
+
+
+class RayCluster():
+    def __init__(self, config: Dict[str, Any]):
+        self.config = config
+        self.name = self.config["cluster_name"]
+        self.config_path = operator_utils.config_path(self.name)
+
+        self.setup_logging()
+
+        self.subprocess = None  # type: Optional[mp.Process]
+
+    def do_in_subprocess(self,
+                         f: Callable[[], None],
+                         wait_to_finish: bool = False) -> None:
+        # First stop the subprocess if it's alive
+        self.clean_up_subprocess()
+        # Reinstantiate process with f as target and start.
+        self.subprocess = mp.Process(name=self.name, target=f)
+        # Kill subprocess if monitor dies
+        self.subprocess.daemon = True
+        self.subprocess.start()
+        if wait_to_finish:
+            self.subprocess.join()
+
+    def clean_up_subprocess(self):
+        if self.subprocess and self.subprocess.is_alive():
+            self.subprocess.terminate()
+            self.subprocess.join()
+
+    def create_or_update(self) -> None:
+        self.do_in_subprocess(self._create_or_update)
+
+    def _create_or_update(self) -> None:
+        self.start_head()
+        self.start_monitor()
+
+    def start_head(self) -> None:
+        self.write_config()
+        self.config = commands.create_or_update_cluster(
+            self.config_path,
+            override_min_workers=None,
+            override_max_workers=None,
+            no_restart=False,
+            restart_only=False,
+            yes=True,
+            no_config_cache=True)
+        self.write_config()
+
+    def start_monitor(self) -> None:
+        ray_head_pod_ip = commands.get_head_node_ip(self.config_path)
+        # TODO: Add support for user-specified redis port and password
+        redis_address = services.address(ray_head_pod_ip,
+                                         ray_constants.DEFAULT_PORT)
+        self.mtr = monitor.Monitor(
+            redis_address=redis_address,
+            autoscaling_config=self.config_path,
+            redis_password=ray_constants.REDIS_DEFAULT_PASSWORD,
+            prefix_cluster_info=True)
+        self.mtr.run()
+
+    def clean_up(self) -> None:
+        self.clean_up_subprocess()
+        self.clean_up_logging()
+        self.delete_config()
+
+    def setup_logging(self) -> None:
+        self.handler = logging.StreamHandler()
+        self.handler.addFilter(lambda rec: rec.processName == self.name)
+        logging_format = ":".join([self.name, ray_constants.LOGGER_FORMAT])
+        self.handler.setFormatter(logging.Formatter(logging_format))
+        operator_utils.root_logger.addHandler(self.handler)
+
+    def clean_up_logging(self) -> None:
+        operator_utils.root_logger.removeHandler(self.handler)
+
+    def write_config(self) -> None:
+        with open(self.config_path, "w") as file:
+            yaml.dump(self.config, file)
+
+    def delete_config(self) -> None:
+        os.remove(self.config_path)
+
+
+ray_clusters = {}
+
+
+def cluster_action(cluster_config: Dict[str, Any], event_type: str) -> None:
+    cluster_name = cluster_config["cluster_name"]
+    if event_type == "ADDED":
+        ray_clusters[cluster_name] = RayCluster(cluster_config)
+        ray_clusters[cluster_name].create_or_update()
+    elif event_type == "MODIFIED":
+        ray_clusters[cluster_name].create_or_update()
+    elif event_type == "DELETED":
+        ray_clusters[cluster_name].clean_up()
+        del ray_clusters[cluster_name]
+
+
+def main() -> None:
+    # Make directory for ray cluster configs
+    if not os.path.isdir(operator_utils.RAY_CONFIG_DIR):
+        os.mkdir(operator_utils.RAY_CONFIG_DIR)
+    # Control loop
+    cluster_cr_stream = operator_utils.cluster_cr_stream()
+    try:
+        for event in cluster_cr_stream:
+            cluster_cr = event["object"]
+            event_type = event["type"]
+            cluster_config = operator_utils.cr_to_config(cluster_cr)
+            cluster_action(cluster_config, event_type)
+    except ApiException as e:
+        if e.status == 404:
+            raise Exception(
+                "Caught a 404 error. Has the RayCluster CRD been created?")
+        else:
+            raise
+
+
+if __name__ == "__main__":
+    main()

python/ray/operator/operator_utils.py

@@ -0,0 +1,114 @@
+import copy
+import logging
+import os
+from typing import Any, Dict, Iterator, List
+
+from kubernetes.watch import Watch
+
+from ray.autoscaler._private.kubernetes import custom_objects_api
+
+RAY_NAMESPACE = os.environ.get("RAY_OPERATOR_POD_NAMESPACE")
+
+RAY_CONFIG_DIR = os.path.expanduser("~/ray_cluster_configs")
+CONFIG_SUFFIX = "_config.yaml"
+
+CONFIG_FIELDS = {
+    "maxWorkers": "max_workers",
+    "upscalingSpeed": "upscaling_speed",
+    "idleTimeoutMinutes": "idle_timeout_minutes",
+    "headPodType": "head_node_type",
+    "workerDefaultPodType": "worker_default_node_type",
+    "workerStartRayCommands": "worker_start_ray_commands",
+    "headStartRayCommands": "head_start_ray_commands",
+    "podTypes": "available_node_types"
+}
+
+NODE_TYPE_FIELDS = {
+    "minWorkers": "min_workers",
+    "maxWorkers": "max_workers",
+    "podConfig": "node_config",
+    "rayResources": "resources",
+    "setupCommands": "worker_setup_commands"
+}
+
+PROVIDER_CONFIG = {
+    "type": "kubernetes",
+    "use_internal_ips": True,
+    "namespace": RAY_NAMESPACE
+}
+
+root_logger = logging.getLogger("ray")
+root_logger.setLevel(logging.getLevelName("DEBUG"))
+"""
+ownerReferences:
+  - apiVersion: apps/v1
+    controller: true
+    blockOwnerDeletion: true
+    kind: ReplicaSet
+    name: my-repset
+    uid: d9607e19-f88f-11e6-a518-42010a800195
+"""
+
+
+def config_path(cluster_name: str) -> str:
+    file_name = cluster_name + CONFIG_SUFFIX
+    return os.path.join(RAY_CONFIG_DIR, file_name)
+
+
+def cluster_cr_stream() -> Iterator:
+    w = Watch()
+    return w.stream(
+        custom_objects_api().list_namespaced_custom_object,
+        namespace=RAY_NAMESPACE,
+        group="cluster.ray.io",
+        version="v1",
+        plural="rayclusters")
+
+
+def cr_to_config(cluster_resource: Dict[str, Any]) -> Dict[str, Any]:
+    """Convert RayCluster custom resource to a ray cluster config for use by the
+    autoscaler."""
+    cr_spec = cluster_resource["spec"]
+    cr_meta = cluster_resource["metadata"]
+    config = translate(cr_spec, dictionary=CONFIG_FIELDS)
+    pod_types = cr_spec["podTypes"]
+    config["available_node_types"] = get_node_types(
+        pod_types, cluster_name=cr_meta["name"], cluster_uid=cr_meta["uid"])
+    config["cluster_name"] = cr_meta["name"]
+    config["provider"] = PROVIDER_CONFIG
+    return config
+
+
+def get_node_types(pod_types: List[Dict[str, Any]], cluster_name: str,
+                   cluster_uid: str) -> Dict[str, Any]:
+    cluster_owner_reference = get_cluster_owner_reference(
+        cluster_name, cluster_uid)
+    node_types = {}
+    for pod_type in pod_types:
+        name = pod_type["name"]
+        pod_type_copy = copy.deepcopy(pod_type)
+        pod_type_copy.pop("name")
+        node_types[name] = translate(
+            pod_type_copy, dictionary=NODE_TYPE_FIELDS)
+        # Deleting a RayCluster CR will also delete the associated pods.
+        node_types[name]["node_config"]["metadata"].update({
+            "ownerReferences": [cluster_owner_reference]
+        })
+    return node_types
+
+
+def get_cluster_owner_reference(cluster_name: str,
+                                cluster_uid: str) -> Dict[str, Any]:
+    return {
+        "apiVersion": "apps/v1",
+        "controller": True,
+        "blockOwnerDeletion": True,
+        "kind": "RayCluster",
+        "name": cluster_name,
+        "uid": cluster_uid
+    }
+
+
+def translate(configuration: Dict[str, Any],
+              dictionary: Dict[str, str]) -> Dict[str, Any]:
+    return {dictionary[field]: configuration[field] for field in configuration}

python/setup.py

@@ -451,7 +451,7 @@ setuptools.setup(
             "ray=ray.scripts.scripts:main",
             "rllib=ray.rllib.scripts:cli [rllib]",
             "tune=ray.tune.scripts:cli",
-            "ray-operator=ray.operator:main",
+            "ray-operator=ray.operator.operator:main",
             "serve=ray.serve.scripts:cli",
         ]
     },