# An unique identifier for the head node and workers of this cluster.
cluster_name: default

# The minimum number of workers nodes to launch in addition to the head
# node. This number should be >= 0.
min_workers: 0

# The maximum number of workers nodes to launch in addition to the head
# node. This takes precedence over min_workers.
max_workers: 2

# The autoscaler will scale up the cluster faster with higher upscaling speed.
# E.g., if the task requires adding more nodes then autoscaler will gradually
# scale up the cluster in chunks of upscaling_speed*currently_running_nodes.
# This number should be > 0.
upscaling_speed: 1.0

# If a node is idle for this many minutes, it will be removed.
idle_timeout_minutes: 1

# Kubernetes resources that need to be configured for the autoscaler to be
# able to manage the Ray cluster. If any of the provided resources don't
# exist, the autoscaler will attempt to create them. If this fails, you may
# not have the required permissions and will have to request them to be
# created by your cluster administrator.
provider:
    type: kubernetes

    # Exposing external IP addresses for ray pods isn't currently supported.
    use_internal_ips: true

    # Namespace to use for all resources created.
    namespace: ray

    # ServiceAccount created by the autoscaler for the head node pod that it
    # runs in. If this field isn't provided, the head pod config below must
    # contain a user-created service account with the proper permissions.
    autoscaler_service_account:
        apiVersion: v1
        kind: ServiceAccount
        metadata:
            name: autoscaler

    # Role created by the autoscaler for the head node pod that it runs in.
    # If this field isn't provided, the role referenced in
    # autoscaler_role_binding must exist and have at least these permissions.
    autoscaler_role:
        kind: Role
        apiVersion: rbac.authorization.k8s.io/v1
        metadata:
            name: autoscaler
        rules:
        - apiGroups: [""]
          resources: ["pods", "pods/status", "pods/exec", "services"]
          verbs: ["get", "watch", "list", "create", "delete", "patch"]
        - apiGroups: ["extensions"]
          resources: ["ingresses"]
          verbs: ["get", "watch", "list", "create", "delete", "patch"]

    # RoleBinding created by the autoscaler for the head node pod that it runs
    # in. If this field isn't provided, the head pod config below must contain
    # a user-created service account with the proper permissions.
    autoscaler_role_binding:
        apiVersion: rbac.authorization.k8s.io/v1
        kind: RoleBinding
        metadata:
            name: autoscaler
        subjects:
        - kind: ServiceAccount
          name: autoscaler
        roleRef:
            kind: Role
            name: autoscaler
            apiGroup: rbac.authorization.k8s.io

    services:
      # Service that maps to the head node of the Ray cluster.
      - apiVersion: v1
        kind: Service
        metadata:
            # NOTE: If you're running multiple Ray clusters with services
            # on one Kubernetes cluster, they must have unique service
            # names.
            name: ray-head
        spec:
            # This selector must match the head node pod's selector below.
            selector:
                component: ray-head
            ports:
                - protocol: TCP
                  port: 8000
                  targetPort: 8000

      # Service that maps to the worker nodes of the Ray cluster.
      - apiVersion: v1
        kind: Service
        metadata:
            # NOTE: If you're running multiple Ray clusters with services
            # on one Kubernetes cluster, they must have unique service
            # names.
            name: ray-workers
        spec:
            # This selector must match the worker node pods' selector below.
            selector:
                component: ray-worker
            ports:
                - protocol: TCP
                  port: 8000
                  targetPort: 8000

# Kubernetes pod config for the head node pod.
head_node:
    pod:
        apiVersion: v1
        kind: Pod
        metadata:
            # Automatically generates a name for the pod with this prefix.
            generateName: ray-head-

            # Must match the head node service selector above if a head node
            # service is required.
            labels:
                component: ray-head
        spec:
            # Change this if you altered the autoscaler_service_account above
            # or want to provide your own.
            serviceAccountName: autoscaler

            # Restarting the head node automatically is not currently supported.
            # If the head node goes down, `ray up` must be run again.
            restartPolicy: Never

            # This volume allocates shared memory for Ray to use for its plasma
            # object store. If you do not provide this, Ray will fall back to
            # /tmp which cause slowdowns if is not a shared memory volume.
            volumes:
            - name: dshm
              emptyDir:
                  medium: Memory

            containers:
            - name: ray-node
              imagePullPolicy: IfNotPresent
              # You are free (and encouraged) to use your own container image,
              # but it should have the following installed:
              #   - rsync (used for `ray rsync` commands and file mounts)
              #   - screen (used for `ray attach`)
              #   - kubectl (used by the autoscaler to manage worker pods)
              image: rayproject/ray:nightly
              # Do not change this command - it keeps the pod alive until it is
              # explicitly killed.
              command: ["/bin/bash", "-c", "--"]
              args: ["trap : TERM INT; sleep infinity & wait;"]
              ports:
                  - containerPort: 6379 # Redis port.
                  - containerPort: 6380 # Redis port.
                  - containerPort: 6381 # Redis port.
                  - containerPort: 12345 # Ray internal communication.
                  - containerPort: 12346 # Ray internal communication.

              # This volume allocates shared memory for Ray to use for its plasma
              # object store. If you do not provide this, Ray will fall back to
              # /tmp which cause slowdowns if is not a shared memory volume.
              volumeMounts:
                  - mountPath: /dev/shm
                    name: dshm
              resources:
                  requests:
                      cpu: 1000m
                      memory: 512Mi
                  limits:
                      # The maximum memory that this pod is allowed to use. The
                      # limit will be detected by ray and split to use 10% for
                      # redis, 30% for the shared memory object store, and the
                      # rest for application memory. If this limit is not set and
                      # the object store size is not set manually, ray will
                      # allocate a very large object store in each pod that may
                      # cause problems for other pods.
                      memory: 2Gi
              env:
                  # This is used in the head_start_ray_commands below so that
                  # Ray can spawn the correct number of processes. Omitting this
                  # may lead to degraded performance.
                  - name: MY_CPU_REQUEST
                    valueFrom:
                        resourceFieldRef:
                            resource: requests.cpu


# Kubernetes pod config for worker node pods.
worker_nodes:
    pod:
        apiVersion: v1
        kind: Pod
        metadata:
            # Automatically generates a name for the pod with this prefix.
            generateName: ray-worker-

            # Must match the worker node service selector above if a worker node
            # service is required.
            labels:
                component: ray-worker
        spec:
            serviceAccountName: default

            # Worker nodes will be managed automatically by the head node, so
            # do not change the restart policy.
            restartPolicy: Never

            # This volume allocates shared memory for Ray to use for its plasma
            # object store. If you do not provide this, Ray will fall back to
            # /tmp which cause slowdowns if is not a shared memory volume.
            volumes:
            - name: dshm
              emptyDir:
                  medium: Memory

            containers:
            - name: ray-node
              imagePullPolicy: IfNotPresent
              # You are free (and encouraged) to use your own container image,
              # but it should have the following installed:
              #   - rsync (used for `ray rsync` commands and file mounts)
              image: rayproject/ray:nightly
              # Do not change this command - it keeps the pod alive until it is
              # explicitly killed.
              command: ["/bin/bash", "-c", "--"]
              args: ["trap : TERM INT; sleep infinity & wait;"]
              ports:
                  - containerPort: 12345 # Ray internal communication.
                  - containerPort: 12346 # Ray internal communication.

              # This volume allocates shared memory for Ray to use for its plasma
              # object store. If you do not provide this, Ray will fall back to
              # /tmp which cause slowdowns if is not a shared memory volume.
              volumeMounts:
                  - mountPath: /dev/shm
                    name: dshm
              resources:
                  requests:
                      cpu: 1000m
                      memory: 512Mi
                  limits:
                      # This memory limit will be detected by ray and split into
                      # 30% for plasma, and 70% for workers.
                      memory: 2Gi
              env:
                  # This is used in the head_start_ray_commands below so that
                  # Ray can spawn the correct number of processes. Omitting this
                  # may lead to degraded performance.
                  - name: MY_CPU_REQUEST
                    valueFrom:
                        resourceFieldRef:
                            resource: requests.cpu

    service:
        apiVersion: v1
        kind: Service
        # The service name gets automatically set by the
        # autoscaler and gets the same name as the pod.
        spec:
            # The right selector is automatically applied by autoscaler
            ports:
                - protocol: TCP
                  port: 8000
                  targetPort: 8000
    
    ingress:
        apiVersion: extensions/v1beta1
        kind: Ingress
        metadata:
          annotations:
            kubernetes.io/ingress.class: nginx
        spec:
          rules:
            - host: localhost
              http:
                paths:
                - path: /
                  backend:
                    # The value of the serviceName must be set to `${RAY_POD_NAME} and will be
                    # automatically replaced by the name of the pod.
                    serviceName: ${RAY_POD_NAME}
                    servicePort: 8000

# Files or directories to copy to the head and worker nodes. The format is a
# dictionary from REMOTE_PATH: LOCAL_PATH, e.g.
file_mounts: {
#    "~/path1/on/remote/machine": "/path1/on/local/machine",
#    "~/path2/on/remote/machine": "/path2/on/local/machine",
}
# Note that the container images in this example have a non-root user.
# To avoid permissions issues, we recommend mounting into a subdirectory of home (~).

# List of commands that will be run before `setup_commands`. If docker is
# enabled, these commands will run outside the container and before docker
# is setup.
initialization_commands: []

# List of shell commands to run to set up nodes.
setup_commands: []

# Custom commands that will be run on the head node after common setup.
head_setup_commands: []

# Custom commands that will be run on worker nodes after common setup.
worker_setup_commands: []

# Command to start ray on the head node. You don't need to change this.
# Note webui-host is set to 0.0.0.0 so that kubernetes can port forward.
head_start_ray_commands:
    - ray stop
    - ulimit -n 65536; ray start --head --num-cpus=$MY_CPU_REQUEST --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --dashboard-host 0.0.0.0

# Command to start ray on worker nodes. You don't need to change this.
worker_start_ray_commands:
    - ray stop
    - ulimit -n 65536; ray start --num-cpus=$MY_CPU_REQUEST --address=$RAY_HEAD_IP:6379 --object-manager-port=8076
