From 9c2c9522626a7cfb168ce5634c9e3ea7ba60363d Mon Sep 17 00:00:00 2001 From: Stephanie Wang Date: Tue, 1 Sep 2020 08:13:04 -0700 Subject: [PATCH] Add documentation for Dask-on-Ray (#10452) * Dask on ray * Dask on ray * line * Add links --- doc/source/dask-on-ray.rst | 43 ++++++++++++++++++++++++++++++ doc/source/index.rst | 1 + doc/source/memory-management.rst | 2 ++ doc/source/ray-overview/basics.rst | 4 ++- doc/source/starting-ray.rst | 2 ++ 5 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 doc/source/dask-on-ray.rst diff --git a/doc/source/dask-on-ray.rst b/doc/source/dask-on-ray.rst new file mode 100644 index 000000000..aa0be4cfb --- /dev/null +++ b/doc/source/dask-on-ray.rst @@ -0,0 +1,43 @@ +Dask on Ray +=========== + +Ray offers an experimental scheduler backend for Dask. +With this plugin, you can use familiar Dask APIs such as Dask DataFrames, and the computation will be executed by the Ray system. + +The Ray plugin can be used with any Dask `.compute() `__ call. +Note that for execution on a Ray cluster, you should *not* use the `Dask.distributed `__ client. +Just follow the instructions for :ref:`using Ray on a cluster ` to modify the ``ray.init()`` call. +Here's an example: + +.. code-block:: python + + import ray + from ray.experimental.dask import ray_dask_get + import dask.delayed + from time import sleep + + # Start Ray. + # Tip: If you're connecting to an existing cluster, use ray.init(address="auto"). + ray.init() + + + def inc(x): + sleep(1) + return x + 1 + + def add(x, y): + sleep(1) + return x + y + + x = dask.delayed(inc)(1) + y = dask.delayed(inc)(2) + z = dask.delayed(add)(x, y) + # The Dask scheduler submits the recorded task graph to Ray. + z.compute(scheduler=ray_dask_get) + +Why use this feature? + + 1. If you'd like to use Dask and Ray libraries in the same application. + 2. To take advantage of Ray-specific features such as the :ref:`cluster launcher ` and :ref:`shared-memory store `. + +Note that Dask-on-Ray is an ongoing project and is not expected to achieve the same performance as using Ray directly. diff --git a/doc/source/index.rst b/doc/source/index.rst index e25bf380d..ccbcc4d26 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -207,6 +207,7 @@ Academic Papers joblib.rst iter.rst pandas_on_ray.rst + dask-on-ray.rst .. toctree:: :hidden: diff --git a/doc/source/memory-management.rst b/doc/source/memory-management.rst index 30d98ea8e..d15550f40 100644 --- a/doc/source/memory-management.rst +++ b/doc/source/memory-management.rst @@ -1,3 +1,5 @@ +.. _memory: + Memory Management ================= diff --git a/doc/source/ray-overview/basics.rst b/doc/source/ray-overview/basics.rst index 84de84166..b9148149b 100644 --- a/doc/source/ray-overview/basics.rst +++ b/doc/source/ray-overview/basics.rst @@ -26,6 +26,8 @@ On top of **Ray Core** are several libraries for solving problems in machine lea Ray also has a number of other community contributed libraries: +- :doc:`../dask-on-ray` +- `Mars on Ray `__ - :doc:`../pandas_on_ray` - :doc:`../joblib` -- :doc:`../multiprocessing` \ No newline at end of file +- :doc:`../multiprocessing` diff --git a/doc/source/starting-ray.rst b/doc/source/starting-ray.rst index aab20485f..a3cfd251a 100644 --- a/doc/source/starting-ray.rst +++ b/doc/source/starting-ray.rst @@ -43,6 +43,8 @@ To check if Ray is initialized, you can call ``ray.is_initialized()``: See the `Configuration `__ documentation for the various ways to configure Ray. +.. _using-ray-on-a-cluster: + Using Ray on a cluster ----------------------