diff --git a/doc/kubernetes/example.py b/doc/kubernetes/example.py index 5ba0272c7..ff2bbe0f4 100644 --- a/doc/kubernetes/example.py +++ b/doc/kubernetes/example.py @@ -3,37 +3,57 @@ from __future__ import division from __future__ import print_function from collections import Counter +import os import sys import time import ray -if __name__ == "__main__": - # Note that if you run this script on a non-head node, then you must - # replace "localhost" with socket.gethostbyname("ray-head"). - ray.init(redis_address="localhost:6379") - # Wait for all 4 nodes to join the cluster. +@ray.remote +def gethostname(x): + import time + import socket + time.sleep(0.01) + return x + (socket.gethostname(), ) + + +def wait_for_nodes(expected): + # Wait for all nodes to join the cluster. while True: num_nodes = len(ray.nodes()) - if num_nodes < 4: - print("{} nodes have joined so far. Waiting for more." - .format(num_nodes)) + if num_nodes < expected: + print("{} nodes have joined so far, waiting for {} more.".format( + num_nodes, expected - num_nodes)) sys.stdout.flush() time.sleep(1) else: break - @ray.remote - def f(x): - time.sleep(0.01) - return x + (ray.services.get_node_ip_address(), ) + +def main(): + wait_for_nodes(4) # Check that objects can be transferred from each node to each other node. - for i in range(100): + for i in range(10): print("Iteration {}".format(i)) - sys.stdout.flush() - print(Counter(ray.get([f.remote(f.remote(())) for _ in range(10000)]))) + results = [ + gethostname.remote(gethostname.remote(())) for _ in range(100) + ] + print(Counter(ray.get(results))) sys.stdout.flush() print("Success!") sys.stdout.flush() + + +if __name__ == "__main__": + # NOTE: If you know you're running this on the head node, you can just + # use "localhost" here. + # redis_host = "localhost" + if ("RAY_HEAD_SERVICE_HOST" not in os.environ + or os.environ["RAY_HEAD_SERVICE_HOST"] == ""): + raise ValueError("RAY_HEAD_SERVICE_HOST environment variable empty." + "Is there a ray cluster running?") + redis_host = os.environ["RAY_HEAD_SERVICE_HOST"] + ray.init(address=redis_host + ":6379") + main() diff --git a/doc/kubernetes/head.yaml b/doc/kubernetes/head.yaml deleted file mode 100644 index a48e9752f..000000000 --- a/doc/kubernetes/head.yaml +++ /dev/null @@ -1,73 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: ray ---- -apiVersion: v1 -kind: Service -metadata: - name: ray-head - namespace: ray -spec: - ports: - - name: redis-primary - port: 6379 - targetPort: 6379 - - name: redis-shard-0 - port: 6380 - targetPort: 6380 - - name: redis-shard-1 - port: 6381 - targetPort: 6381 - - name: object-manager - port: 12345 - targetPort: 12345 - - name: node-manager - port: 12346 - targetPort: 12346 - selector: - component: ray-head ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ray-head - namespace: ray -spec: - replicas: 1 - selector: - matchLabels: - component: ray-head - type: ray - template: - metadata: - labels: - component: ray-head - type: ray - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - type: ray - topologyKey: kubernetes.io/hostname - containers: - - name: ray-head - image: rayproject/examples - command: [ "/bin/bash", "-c", "--" ] - args: ["ray start --head --redis-port=6379 --redis-shard-ports=6380,6381 --object-manager-port=12345 --node-manager-port=12346 --node-ip-address=$MY_POD_IP --block"] - ports: - - containerPort: 6379 - - containerPort: 6380 - - containerPort: 6381 - - containerPort: 12345 - - containerPort: 12346 - env: - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - resources: - requests: - cpu: 2 diff --git a/doc/kubernetes/ray-cluster.yaml b/doc/kubernetes/ray-cluster.yaml new file mode 100644 index 000000000..853f9dfb2 --- /dev/null +++ b/doc/kubernetes/ray-cluster.yaml @@ -0,0 +1,150 @@ +# Ray head node service, allowing worker pods to discover the head node. +apiVersion: v1 +kind: Service +metadata: + namespace: ray + name: ray-head +spec: + ports: + # Redis ports. + - name: redis-primary + port: 6379 + targetPort: 6379 + - name: redis-shard-0 + port: 6380 + targetPort: 6380 + - name: redis-shard-1 + port: 6381 + targetPort: 6381 + + # Ray internal communication ports. + - name: object-manager + port: 12345 + targetPort: 12345 + - name: node-manager + port: 12346 + targetPort: 12346 + selector: + component: ray-head +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: ray + name: ray-head +spec: + # Do not change this - Ray currently only supports one head node per cluster. + replicas: 1 + selector: + matchLabels: + component: ray-head + type: ray + template: + metadata: + labels: + component: ray-head + type: ray + spec: + # If the head node goes down, the entire cluster (including all worker + # nodes) will go down as well. If you want Kubernetes to bring up a new + # head node in this case, set this to "Always," else set it to "Never." + restartPolicy: Always + + # This volume allocates shared memory for Ray to use for its plasma + # object store. If you do not provide this, Ray will fall back to + # /tmp which cause slowdowns if is not a shared memory volume. + volumes: + - name: dshm + emptyDir: + medium: Memory + containers: + - name: ray-head + image: rayproject/autoscaler + imagePullPolicy: Always + command: [ "/bin/bash", "-c", "--" ] + args: + - "ray start --head --node-ip-address=$MY_POD_IP --redis-port=6379 --redis-shard-ports=6380,6381 --num-cpus=$MY_CPU_REQUEST --object-manager-port=12345 --node-manager-port=12346 --block" + ports: + - containerPort: 6379 # Redis port. + - containerPort: 6380 # Redis port. + - containerPort: 6381 # Redis port. + - containerPort: 12345 # Ray internal communication. + - containerPort: 12346 # Ray internal communication. + + # This volume allocates shared memory for Ray to use for its plasma + # object store. If you do not provide this, Ray will fall back to + # /tmp which cause slowdowns if is not a shared memory volume. + volumeMounts: + - mountPath: /dev/shm + name: dshm + env: + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + + # This is used in the ray start command so that Ray can spawn the + # correct number of processes. Omitting this may lead to degraded + # performance. + - name: MY_CPU_REQUEST + valueFrom: + resourceFieldRef: + resource: requests.cpu + resources: + requests: + cpu: 100m + memory: 512Mi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: ray + name: ray-worker +spec: + # Change this to scale the number of worker nodes started in the Ray cluster. + replicas: 3 + selector: + matchLabels: + component: ray-worker + type: ray + template: + metadata: + labels: + component: ray-worker + type: ray + spec: + restartPolicy: Always + volumes: + - name: dshm + emptyDir: + medium: Memory + containers: + - name: ray-worker + image: rayproject/autoscaler + imagePullPolicy: Always + command: ["/bin/bash", "-c", "--"] + args: + - "ray start --node-ip-address=$MY_POD_IP --num-cpus=$MY_CPU_REQUEST --address=$RAY_HEAD_SERVICE_HOST:$RAY_HEAD_SERVICE_PORT_REDIS_PRIMARY --object-manager-port=12345 --node-manager-port=12346 --block" + ports: + - containerPort: 12345 # Ray internal communication. + - containerPort: 12346 # Ray internal communication. + volumeMounts: + - mountPath: /dev/shm + name: dshm + env: + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + + # This is used in the ray start command so that Ray can spawn the + # correct number of processes. Omitting this may lead to degraded + # performance. + - name: MY_CPU_REQUEST + valueFrom: + resourceFieldRef: + resource: requests.cpu + resources: + requests: + cpu: 100m + memory: 512Mi diff --git a/doc/kubernetes/ray-job.yaml b/doc/kubernetes/ray-job.yaml new file mode 100644 index 000000000..5f46198cc --- /dev/null +++ b/doc/kubernetes/ray-job.yaml @@ -0,0 +1,32 @@ +# Job to run a Ray program in its own pod. Assumes that a cluster is already +# running (e.g., from './ray-cluster.yaml'). +apiVersion: batch/v1 +kind: Job +metadata: + namespace: ray + generateName: ray-test-job- +spec: + template: + spec: + restartPolicy: Never + containers: + - name: ray-head + image: rayproject/autoscaler + imagePullPolicy: Always + command: [ "/bin/bash", "-c", "--" ] + args: + - "wget https://raw.githubusercontent.com/ray-project/ray/master/doc/kubernetes/example.py && + ray start --node-ip-address=$MY_POD_IP --num-cpus=0 --redis-address=$RAY_HEAD_SERVICE_HOST:$RAY_HEAD_SERVICE_PORT_REDIS_PRIMARY --object-manager-port=12345 --node-manager-port=12346 && + python example.py" + ports: + - containerPort: 12345 # Ray internal communication. + - containerPort: 12346 # Ray internal communication. + env: + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + resources: + requests: + cpu: 100m + memory: 512Mi diff --git a/doc/kubernetes/ray-namespace.yaml b/doc/kubernetes/ray-namespace.yaml new file mode 100644 index 000000000..3f379c375 --- /dev/null +++ b/doc/kubernetes/ray-namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ray diff --git a/doc/kubernetes/submit.yaml b/doc/kubernetes/submit.yaml deleted file mode 100644 index 129ad34eb..000000000 --- a/doc/kubernetes/submit.yaml +++ /dev/null @@ -1,117 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: ray ---- -apiVersion: v1 -kind: Service -metadata: - name: ray-head - namespace: ray -spec: - ports: - - name: redis-primary - port: 6379 - targetPort: 6379 - - name: redis-shard-0 - port: 6380 - targetPort: 6380 - - name: redis-shard-1 - port: 6381 - targetPort: 6381 - - name: object-manager - port: 12345 - targetPort: 12345 - - name: node-manager - port: 12346 - targetPort: 12346 - selector: - component: ray-head ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ray-head - namespace: ray -spec: - replicas: 1 - selector: - matchLabels: - component: ray-head - type: ray - template: - metadata: - labels: - component: ray-head - type: ray - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - type: ray - topologyKey: kubernetes.io/hostname - containers: - - name: ray-head - image: rayproject/examples - command: [ "/bin/bash", "-c", "--" ] - args: - - "wget https://raw.githubusercontent.com/ray-project/ray/master/doc/kubernetes/example.py && - ray start --head --redis-port=6379 --redis-shard-ports=6380,6381 --object-manager-port=12345 --node-manager-port=12346 --node-ip-address=$MY_POD_IP && - python example.py" - ports: - - containerPort: 6379 - - containerPort: 6380 - - containerPort: 6381 - - containerPort: 12345 - - containerPort: 12346 - env: - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - resources: - requests: - cpu: 2 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ray-worker - namespace: ray -spec: - replicas: 3 - selector: - matchLabels: - component: ray-worker - type: ray - template: - metadata: - labels: - component: ray-worker - type: ray - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - type: ray - topologyKey: kubernetes.io/hostname - containers: - - name: ray-worker - image: rayproject/examples - command: ["/bin/bash", "-c", "--"] - args: ["ray start --node-ip-address=$MY_POD_IP --redis-address=$(python -c 'import socket;import sys; sys.stdout.write(socket.gethostbyname(\"ray-head\"));sys.stdout.flush()'):6379 --object-manager-port=12345 --node-manager-port=12346 --block"] - ports: - - containerPort: 12345 - - containerPort: 12346 - env: - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - resources: - requests: - cpu: 2 diff --git a/doc/kubernetes/worker.yaml b/doc/kubernetes/worker.yaml deleted file mode 100644 index 2b0399e2b..000000000 --- a/doc/kubernetes/worker.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ray-worker - namespace: ray -spec: - replicas: 3 - selector: - matchLabels: - component: ray-worker - type: ray - template: - metadata: - labels: - component: ray-worker - type: ray - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - type: ray - topologyKey: kubernetes.io/hostname - containers: - - name: ray-worker - image: rayproject/examples - command: ["/bin/bash", "-c", "--"] - args: ["ray start --node-ip-address=$MY_POD_IP --redis-address=$(python -c 'import socket;import sys; sys.stdout.write(socket.gethostbyname(\"ray-head\"));sys.stdout.flush()'):6379 --object-manager-port=12345 --node-manager-port=12346 --block"] - ports: - - containerPort: 12345 - - containerPort: 12346 - env: - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - resources: - requests: - cpu: 2 diff --git a/doc/source/deploy-on-kubernetes.rst b/doc/source/deploy-on-kubernetes.rst index 78e664b65..9a00f3218 100644 --- a/doc/source/deploy-on-kubernetes.rst +++ b/doc/source/deploy-on-kubernetes.rst @@ -1,148 +1,307 @@ Deploying on Kubernetes ======================= +.. note:: + + The easiest way to run a Ray cluster is by using the built-in autoscaler, + which has support for running on top of Kubernetes. Please see the `autoscaler + documentation `__ for details. + .. warning:: - These instructions have not been tested extensively. If you have a suggestion - for how to improve them, please open a pull request or email - ray-dev@googlegroups.com. + Running Ray on Kubernetes is still a work in progress. If you have a + suggestion for how to improve them or want to request a missing feature, + please get in touch using one of the channels in the + `Questions or Issues?`_ section below. -You can run Ray on top of Kubernetes. This document assumes that you have access -to a Kubernetes cluster and have ``kubectl`` installed locally. +This document assumes that you have access to a Kubernetes cluster and have +``kubectl`` installed locally and configured to access the cluster. It will +first walk you through how to deploy a Ray cluster on your existing Kubernetes +cluster, then explore a few different ways to run programs on the Ray cluster. -Start by cloning the Ray repository. +The configuration ``yaml`` files here are provided as examples to get you +started. At a minimum you will probably want to build and use your own +container images, add more worker nodes to the cluster (or use the +`Kubernetes Horizontal Pod Autoscaler`_), and change the resource requests for +the head and worker nodes. Refer to the provided ``yaml`` files to be sure that +you maintain important configuration options for Ray to function properly. + +Creating a Ray Namespace +------------------------ + +First, create a `Kubernetes Namespace`_ for Ray resources on your cluster. The +following commands will create resources under this Namespace, so if you want +to use a different one than ``ray``, please be sure to also change the +`namespace` fields in the provided ``yaml`` files and anytime you see a ``-n`` +flag passed to ``kubectl``. .. code-block:: shell - git clone https://github.com/ray-project/ray.git + $ kubectl create -f ray/doc/kubernetes/ray-namespace.yaml -Work Interactively on the Cluster ---------------------------------- +Starting a Ray Cluster +---------------------- -To work interactively, first start Ray on Kubernetes. +A Ray cluster consists of a single head node and a set of worker nodes (the +provided ``ray-cluster.yaml`` file will start 3 worker nodes). In the example +Kubernetes configuration, this is implemented as: + +- A ``ray-head`` `Kubernetes Service`_ that enables the worker nodes to discover the location of the head node on start up. +- A ``ray-head`` `Kubernetes Deployment`_ that backs the ``ray-head`` Service with a single head node pod (replica). +- A ``ray-worker`` `Kubernetes Deployment`_ with multiple worker node pods (replicas) that connect to the ``ray-head`` pod using the ``ray-head`` Service. + +Note that because the head and worker nodes are Deployments, Kubernetes will +automatically restart pods that crash to maintain the correct number of +replicas. + +- If a worker node goes down, a replacement pod will be started and joined to the cluster. +- If the head node goes down, it will be restarted. This will start a new Ray cluster. Worker nodes that were connected to the old head node will crash and be restarted, connecting to the new head node when they come back up. + +Try deploying a cluster with the provided Kubernetes config by running the +following command: .. code-block:: shell - kubectl create -f ray/kubernetes/head.yaml - kubectl create -f ray/kubernetes/worker.yaml + $ kubectl apply -f ray/doc/kubernetes/ray-cluster.yaml -This will start one head pod and 3 worker pods. You can check that the pods are -running by running ``kubectl get pods -n ray``. - -You should see something like the following (you will have to wait a couple -minutes for the pods to enter the "Running" state). +Verify that the pods are running by running ``kubectl get pods -n ray``. You +may have to wait up to a few minutes for the pods to enter the 'Running' +state on the first run. .. code-block:: shell - $ kubectl get pods -n ray + $ kubectl -n ray get pods NAME READY STATUS RESTARTS AGE ray-head-5455bb66c9-6bxvz 1/1 Running 0 10s ray-worker-5c49b7cc57-c6xs8 1/1 Running 0 5s ray-worker-5c49b7cc57-d9m86 1/1 Running 0 5s ray-worker-5c49b7cc57-kzk4s 1/1 Running 0 5s -To run tasks interactively on the cluster, connect to one of the pods, e.g., +.. note:: + + You might see a nonzero number of RESTARTS for the worker pods. That can + happen when the worker pods start up before the head pod and the workers + aren't able to connect. This shouldn't affect the behavior of the cluster. + +To change the number of worker nodes in the cluster, change the ``replicas`` +field in the worker deployment configuration in that file and then re-apply +the config as follows: .. code-block:: shell - kubectl exec -it -n ray ray-head-5455bb66c9-6bxvz -- bash + # Edit 'ray/doc/kubernetes/ray-cluster.yaml' and change the 'replicas' + # field under the ray-worker deployment to, e.g., 4. -Start an IPython interpreter, e.g., ``ipython`` + # Re-apply the new configuration to the running deployment. + $ kubectl apply -f ray/doc/kubernetes/ray-cluster.yaml + service/ray-head unchanged + deployment.apps/ray-head unchanged + deployment.apps/ray-worker configured + + # Verify that there are now the correct number of worker pods running. + $ kubectl -n ray get pods + NAME READY STATUS RESTARTS AGE + ray-head-5455bb66c9-6bxvz 1/1 Running 0 30s + ray-worker-5c49b7cc57-c6xs8 1/1 Running 0 25s + ray-worker-5c49b7cc57-d9m86 1/1 Running 0 25s + ray-worker-5c49b7cc57-kzk4s 1/1 Running 0 25s + ray-worker-5c49b7cc57-zzfg2 1/1 Running 0 0s + +To validate that the restart behavior is working properly, try killing pods +and checking that they are restarted by Kubernetes: + +.. code-block:: shell + + # Delete a worker pod. + $ kubectl -n ray delete ray-worker-5c49b7cc57-c6xs8 + pod "ray-worker-5c49b7cc57-c6xs8" deleted + + # Check that a new worker pod was started (this may take a few seconds). + $ kubectl -n ray get pods + NAME READY STATUS RESTARTS AGE + ray-head-5455bb66c9-6bxvz 1/1 Running 0 45s + ray-worker-5c49b7cc57-d9m86 1/1 Running 0 40s + ray-worker-5c49b7cc57-kzk4s 1/1 Running 0 40s + ray-worker-5c49b7cc57-ypq8x 1/1 Running 0 0s + + # Delete the head pod. + $ kubectl -n ray delete ray-head-5455bb66c9-6bxvz + pod "ray-head-5455bb66c9-6bxvz" deleted + + # Check that a new head pod was started and the worker pods were restarted. + $ kubectl -n ray get pods + NAME READY STATUS RESTARTS AGE + ray-head-5455bb66c9-gqzql 1/1 Running 0 0s + ray-worker-5c49b7cc57-d9m86 1/1 Running 1 50s + ray-worker-5c49b7cc57-kzk4s 1/1 Running 1 50s + ray-worker-5c49b7cc57-ypq8x 1/1 Running 1 10s + + # You can even try deleting all of the pods in the Ray namespace and checking + # that Kubernetes brings the right number back up. + $ kubectl -n ray delete pods --all + $ kubectl -n ray get pods + NAME READY STATUS RESTARTS AGE + ray-head-5455bb66c9-7l6xj 1/1 Running 0 10s + ray-worker-5c49b7cc57-57tpv 1/1 Running 0 10s + ray-worker-5c49b7cc57-6m4kp 1/1 Running 0 10s + ray-worker-5c49b7cc57-jx2w2 1/1 Running 0 10s + +Running Ray Programs +-------------------- + +This section assumes that you have a running Ray cluster (if you don't, please +refer to the section above to get started) and will walk you through three +different options to run a Ray program on it: + +1. Using `kubectl exec` to run a Python script. +2. Using `kubectl exec -it bash` to work interactively in a remote shell. +3. Submitting a `Kubernetes Job`_. + +Running a program using 'kubectl exec' +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To run an example program that tests object transfers between nodes in the +cluster, try the following commands (don't forget to replace the head pod name +- you can find it by running ``kubectl -n ray get pods``): + +.. code-block:: shell + + # Copy the test script onto the head node. + $ kubectl -n ray cp ray/doc/kubernetes/example.py ray-head-5455bb66c9-7l6xj:/example.py + + # Run the example program on the head node. + $ kubectl -n ray exec ray-head-5455bb66c9-7l6xj -- python example.py + # You should see repeated output for 10 iterations and then 'Success!' + +Running a program in a remote shell +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can also run tasks interactively on the cluster by connecting a remote +shell to one of the pods. + +.. code-block:: shell + + # Copy the test script onto the head node. + $ kubectl -n ray cp ray/doc/kubernetes/example.py ray-head-5455bb66c9-7l6xj:/example.py + + # Get a remote shell to the head node. + $ kubectl -n ray exec -it ray-head-5455bb66c9-7l6xj -- bash + + # Run the example program on the head node. + root@ray-head-6f566446c-5rdmb:/# python example.py + # You should see repeated output for 10 iterations and then 'Success!' + +You can also start an IPython interpreter to work interactively: + +.. code-block:: shell + + # From your local machine. + $ kubectl -n ray exec -it ray-head-5455bb66c9-7l6xj -- ipython + + # From a remote shell on the head node. + $ kubectl -n ray exec -it ray-head-5455bb66c9-7l6xj -- bash + root@ray-head-6f566446c-5rdmb:/# ipython + +Once you have the IPython interpreter running, try running the following example +program: .. code-block:: python from collections import Counter + import socket import time import ray - # Note that if you run this script on a non-head node, then you must replace - # "localhost" with socket.gethostbyname("ray-head"). - ray.init(address="localhost:6379") + ray.init(address="$RAY_HEAD_SERVICE_HOST:$RAY_HEAD_SERVICE_PORT_REDIS_PRIMARY") @ray.remote def f(x): time.sleep(0.01) - return x + (ray.services.get_node_ip_address(), ) + return x + (socket.gethostname(), ) # Check that objects can be transferred from each node to each other node. - %time Counter(ray.get([f.remote(f.remote(())) for _ in range(1000)])) + %time Counter(ray.get([f.remote(f.remote(())) for _ in range(100)])) -Submitting a Script to the Cluster ----------------------------------- +Submitting a Job +~~~~~~~~~~~~~~~~ -To submit a self-contained Ray application to your Kubernetes cluster, do the -following. +You can also submit a Ray application to run on the cluster as a `Kubernetes +Job`_. The Job will run a single pod running the Ray driver program to +completion, then terminate the pod but allow you to access the logs. + +To submit a Job that downloads and executes an `example program`_ that tests +object transfers between nodes in the cluster, run the following command: .. code-block:: shell - kubectl create -f ray/kubernetes/submit.yaml + $ kubectl create -f ray/doc/kubernetes/ray-job.yaml + job.batch/ray-test-job-kw5gn created -One of the pods will download and run `this example script`_. +.. _`example program`: https://github.com/ray-project/ray/blob/master/doc/kubernetes/example.py -.. _`this example script`: https://github.com/ray-project/ray/tree/master/doc/kubernetes/example.py - -The script prints its output. To view the output, first find the pod name by -running ``kubectl get all``. You'll see output like the following. +To view the output of the Job, first find the name of the pod that ran it, +then fetch its logs: .. code-block:: shell - $ kubectl get all -n ray - NAME READY STATUS RESTARTS AGE - pod/ray-head-5486648dc9-c6hz2 1/1 Running 0 11s - pod/ray-worker-5c49b7cc57-2jz4l 1/1 Running 0 11s - pod/ray-worker-5c49b7cc57-8nwjk 1/1 Running 0 11s - pod/ray-worker-5c49b7cc57-xlksn 1/1 Running 0 11s + $ kubectl -n ray get pods + NAME READY STATUS RESTARTS AGE + ray-head-5455bb66c9-7l6xj 1/1 Running 0 15s + ray-test-job-kw5gn-5g7tv 0/1 Completed 0 10s + ray-worker-5c49b7cc57-57tpv 1/1 Running 0 15s + ray-worker-5c49b7cc57-6m4kp 1/1 Running 0 15s + ray-worker-5c49b7cc57-jx2w2 1/1 Running 0 15s - NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE - service/ray-head ClusterIP 10.110.54.241 6379/TCP,6380/TCP,6381/TCP,12345/TCP,12346/TCP 11s + # Fetch the logs. You should see repeated output for 10 iterations and then + # 'Success!' + $ kubectl -n ray logs ray-test-job-kw5gn-5g7tv - NAME READY UP-TO-DATE AVAILABLE AGE - deployment.apps/ray-head 1/1 1 1 11s - deployment.apps/ray-worker 3/3 3 3 11s - - NAME DESIRED CURRENT READY AGE - replicaset.apps/ray-head-5486648dc9 1 1 1 11s - replicaset.apps/ray-worker-5c49b7cc57 3 3 3 11s - -Find the name of the ``ray-head`` pod and run the equivalent of +To clean up the resources created by the Job after checking its output, run +the following: .. code-block:: shell - kubectl logs ray-head-5486648dc9-c6hz2 -n ray + # List Jobs run in the Ray namespace. + $ kubectl -n ray get jobs + NAME COMPLETIONS DURATION AGE + ray-test-job-kw5gn 1/1 10s 30s + + # Delete the finished Job. + $ kubectl -n ray delete job ray-test-job-kw5gn + + # Verify that the Job's pod was cleaned up. + $ kubectl -n ray get pods + NAME READY STATUS RESTARTS AGE + ray-head-5455bb66c9-7l6xj 1/1 Running 0 60s + ray-worker-5c49b7cc57-57tpv 1/1 Running 0 60s + ray-worker-5c49b7cc57-6m4kp 1/1 Running 0 60s + ray-worker-5c49b7cc57-jx2w2 1/1 Running 0 60s Cleaning Up ----------- -To remove the services you have created, run the following. +To delete a running Ray cluster, you can run the following command: .. code-block:: shell - kubectl delete -n ray service/ray-head \ - deployment.apps/ray-head \ - deployment.apps/ray-worker + kubectl delete -f ray/doc/kubernetes/ray-cluster.yaml +Questions or Issues? +-------------------- -Customization -------------- +You can post questions or issues or feedback through the following channels: -You will probably need to do some amount of customization. +1. `ray-dev@googlegroups.com`_: For discussions about development or any general + questions and feedback. +2. `StackOverflow`_: For questions about how to use Ray. +3. `GitHub Issues`_: For bug reports and feature requests. -1. The example above uses the Docker image ``rayproject/examples``, which is - built using `these Dockerfiles`_. You will most likely need to use your own - Docker image. -2. You will need to modify the ``command`` and ``args`` fields to potentially - install and run the script of your choice. -3. You will need to customize the resource requests. +.. _`ray-dev@googlegroups.com`: https://groups.google.com/forum/#!forum/ray-dev +.. _`StackOverflow`: https://stackoverflow.com/questions/tagged/ray +.. _`GitHub Issues`: https://github.com/ray-project/ray/issues -TODO ----- - -The following are also important but haven't been documented yet. Contributions -are welcome! - -1. Request CPU/GPU/memory resources. -2. Increase shared memory. -3. How to make Kubernetes clean itself up once the script finishes. -4. Follow Kubernetes best practices. - -.. _`these Dockerfiles`: https://github.com/ray-project/ray/tree/master/docker +.. _`Kubernetes Horizontal Pod Autoscaler`: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/ +.. _`Kubernetes Namespace`: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ +.. _`Kubernetes Service`: https://kubernetes.io/docs/concepts/services-networking/service/ +.. _`Kubernetes Deployment`: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/ +.. _`Kubernetes Job`: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/