diff --git a/.travis.yml b/.travis.yml index 853e83b74..4c9a4866d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,14 @@ matrix: install: [] script: - .travis/check-git-clang-format-output.sh + # Try generating Sphinx documentation. To do this, we need to setup some + # Python stuff. + - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + - bash miniconda.sh -b -p $HOME/miniconda + - export PATH="$HOME/miniconda/bin:$PATH" + - cd doc + - pip install -r requirements-doc.txt + - sphinx-build -W -b html -d _build/doctrees source _build/html - os: linux dist: trusty env: VALGRIND=1 PYTHON=2.7 diff --git a/README.md b/README.md index afabb51ed..9e9a43d64 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # Ray [![Build Status](https://travis-ci.org/ray-project/ray.svg?branch=master)](https://travis-ci.org/ray-project/ray) +[![Documentation Status](https://readthedocs.org/projects/ray/badge/?version=latest)](http://ray.readthedocs.io/en/latest/?badge=latest) Ray is an experimental distributed execution engine. It is under development and not ready to be used. @@ -9,58 +10,4 @@ The goal of Ray is to make it easy to write machine learning applications that run on a cluster while providing the development and debugging experience of working on a single machine. -Before jumping into the details, here's a simple Python example for doing a -Monte Carlo estimation of pi (using multiple cores or potentially multiple -machines). - -```python -import ray -import numpy as np - -# Start Ray with some workers. -ray.init(num_workers=10) - -# Define a remote function for estimating pi. -@ray.remote -def estimate_pi(n): - x = np.random.uniform(size=n) - y = np.random.uniform(size=n) - return 4 * np.mean(x ** 2 + y ** 2 < 1) - -# Launch 10 tasks, each of which estimates pi. -result_ids = [] -for _ in range(10): - result_ids.append(estimate_pi.remote(100)) - -# Fetch the results of the tasks and print their average. -estimate = np.mean(ray.get(result_ids)) -print("Pi is approximately {}.".format(estimate)) -``` - -Within the for loop, each call to `estimate_pi.remote(100)` sends a message to -the scheduler asking it to schedule the task of running `estimate_pi` with the -argument `100`. This call returns right away without waiting for the actual -estimation of pi to take place. Instead of returning a float, it returns an -**object ID**, which represents the eventual output of the computation (this is -a similar to a Future). - -The call to `ray.get(result_id)` takes an object ID and returns the actual -estimate of pi (waiting until the computation has finished if necessary). - -## Next Steps - -- Installation on [Ubuntu](doc/install-on-ubuntu.md), [Mac OS X](doc/install-on-macosx.md) - - [Troubleshooting](doc/installation-troubleshooting.md) -- [Tutorial](doc/tutorial.md) -- Documentation - - [Serialization in the Object Store](doc/serialization.md) - - [Environment Variables](doc/environment-variables.md) - - [Using Ray with TensorFlow](doc/using-ray-with-tensorflow.md) - - [Using Ray on a Cluster](doc/using-ray-on-a-cluster.md) - - [Example Management Using Parallel-SSH](doc/using-ray-on-a-large-cluster.md) - -## Example Applications - -- [Hyperparameter Optimization](examples/hyperopt/README.md) -- [Batch L-BFGS](examples/lbfgs/README.md) -- [Learning to Play Pong](examples/rl_pong/README.md) +View the [documentation](http://ray.readthedocs.io/en/latest/index.html). diff --git a/doc/Makefile b/doc/Makefile index 6943059d5..e188f9a06 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -15,9 +15,9 @@ endif # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source # the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 000000000..f001d0d68 --- /dev/null +++ b/doc/README.md @@ -0,0 +1,15 @@ +# Ray Documentation + +To compile the documentation, run the following commands from this directory. + +``` +pip install -r requirements-doc.txt +make html +open _build/html/index.html +``` + +To test if there are any build errors with the documentation, do the following. + +``` +sphinx-build -W -b html -d _build/doctrees source _build/html +``` diff --git a/doc/api.rst b/doc/api.rst deleted file mode 100644 index b6c092dcc..000000000 --- a/doc/api.rst +++ /dev/null @@ -1,15 +0,0 @@ -=========== -The Ray API -=========== - -.. autofunction:: ray.put -.. autofunction:: ray.get -.. autofunction:: ray.remote -.. autofunction:: ray.wait -.. autofunction:: ray.init -.. autofunction:: ray.kill_workers -.. autofunction:: ray.restart_workers_local -.. autofunction:: ray.visualize_computation_graph -.. autofunction:: ray.scheduler_info -.. autofunction:: ray.task_info -.. autofunction:: ray.register_module diff --git a/doc/environment-variables.md b/doc/environment-variables.md deleted file mode 100644 index d695dba19..000000000 --- a/doc/environment-variables.md +++ /dev/null @@ -1,106 +0,0 @@ -# Environment Variables - -This document explains how to create and use **environment variables** in Ray. - -An environment variable is a per-worker variable which is (1) created when the -worker starts, and (2) is reinitialized before a task reuses it. Thus, while a -task can modify an environment variable, the variable is reinitialized before -the next task uses it. Environment variables obviates the need for -serialization/deserialization and help avoid side effects. - -Environment variables are Python objects that are created once on each worker -and can be used by all subsequent tasks that run on that worker. Environment -variables will be reinitialized between tasks. There are several primary reasons -for using environment variables. - -1. Environment variables are created once on each worker and are not shipped -between machines, so they do not need to be serialized or deserialized (however, -the code that creates the environment variable does need to be pickled). -2. Objects that are slow to construct (like a TensorFlow graph) only need to be -constructed once on each worker. -3. By reinitializing between tasks that use them, they help avoid side effects. - -To elaborate on the first point, standard Python serialization libraries like -pickle fail on some objects. With these kinds of objects, it may be easier to -ship the code that creates the object to each worker and to run the code on each -worker than it would be to create the object on the driver and ship the object -to each worker. - -## Creating an Environment Variable - -To give an example, consider a gym environment, which essentially provides a -Python wrapper for an Atari simulator. - -```python -import gym -import ray - -ray.init(num_workers=10) - -# Define a function to create the gym environment. -def env_initializer(): - return gym.make("Pong-v0") - -# Create the environment variable. This line will cause env_initializer to run -# on each worker and on the driver. -ray.env.env = ray.EnvironmentVariable(env_initializer) - -# Define a remote function that uses the gym environment. -@ray.remote -def step(): - env = ray.env.env - # Choose a random action. - action = env.action_space.sample() - # Take the action and return the result. - return env.step(action) - -# Call the remote function. -step.remote() -``` - -When the gym is created, it prints something like `Making new env: Pong-v0`. You -may notice that this is printed once for each worker. Calling `step.remote()` -will run a remote function that uses the `env` variable. You may notice that -calling `step.remote()` causes the line `Making new env: Pong-v0` to be printed -again. That occurs because, by default, every time a remote function uses an -environment variable, the worker will rerun the code that initializes the -environment variable to prevent side effects from leaking between tasks and -introducing non-determinism into the program. - -Of course, rerunning the initialization code can be expensive, so a custom -reinitializer can be passed into the creation of an environment variable. If the -state of the environment variable is not mutated by any remote function, then -the reinitialization code can just be the identity function. - -```python -# Define a function to create the gym environment. -def env_initializer(): - return gym.make("Pong-v0") - -# Define a function to reinitialize the gym environment. -def env_reinitializer(env): - env.reset() - return env - -# Create the environment variable. This line will cause env_initializer to run -# on each worker and on the driver. Every time a remote function uses the -# environment variable, env_reinitializer will run to reset the state of the -# variable. -ray.env.env = ray.EnvironmentVariable(env_initializer, env_reinitializer) - -# Define a remote function that uses the gym environment. -@ray.remote -def step(): - env = ray.env.env - # Choose a random action. - action = env.action_space.sample() - # Take the action and return the result. - return env.step(action) - -# Call the remote function. -step.remote() -``` - -**Note:** It may sometimes look like Ray is hanging and not responding. This can -occur when print statements happen in the background on workers and hide the -interpreter prompt. Try pressing enter, and see if that fixes it. diff --git a/doc/figures/compgraph1.png b/doc/figures/compgraph1.png deleted file mode 100644 index bc34726b0..000000000 Binary files a/doc/figures/compgraph1.png and /dev/null differ diff --git a/doc/figures/compgraph2.png b/doc/figures/compgraph2.png deleted file mode 100644 index 923096304..000000000 Binary files a/doc/figures/compgraph2.png and /dev/null differ diff --git a/doc/index.rst b/doc/index.rst deleted file mode 100644 index c1cdf23d6..000000000 --- a/doc/index.rst +++ /dev/null @@ -1,16 +0,0 @@ -.. Ray documentation master file, created by - sphinx-quickstart on Fri Jul 1 13:19:58 2016. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to Ray's documentation! -=============================== - -API: - -.. toctree:: - :maxdepth: 2 - - api - cluster-api - services-api diff --git a/doc/install-on-windows.md b/doc/install-on-windows.md deleted file mode 100644 index cbd4c9125..000000000 --- a/doc/install-on-windows.md +++ /dev/null @@ -1,26 +0,0 @@ -# Installation on Windows - -Ray currently does not run on Windows. However, it can be compiled with the -following instructions. - -We currently do not support Python 3. - -**Note:** A batch file is provided that clones any missing third-party libraries -and applies patches to them. Do not attempt to open the solution before the -batch file applies the patches; otherwise, if the projects have been modified, -the patches may be rejected, and you may be forced to revert your changes before -re-running the batch file. - -1. Install Microsoft Visual Studio 2015 -2. Install Git -3. `git clone https://github.com/ray-project/ray.git` -4. `ray\thirdparty\download_thirdparty.bat` - -## Test if the installation succeeded - -To test if the installation was successful, try running some tests. - -``` -python test/runtest.py # This tests basic functionality. -python test/array_test.py # This tests some array libraries. -``` diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt new file mode 100644 index 000000000..ab1b03d49 --- /dev/null +++ b/doc/requirements-doc.txt @@ -0,0 +1,10 @@ +colorama +cloudpickle +funcsigs +mock +numpy +psutil +recommonmark +redis +sphinx +sphinx_rtd_theme diff --git a/doc/_static/.gitkeep b/doc/source/_static/.gitkeep similarity index 100% rename from doc/_static/.gitkeep rename to doc/source/_static/.gitkeep diff --git a/doc/_templates/.gitkeep b/doc/source/_templates/.gitkeep similarity index 100% rename from doc/_templates/.gitkeep rename to doc/source/_templates/.gitkeep diff --git a/doc/source/api.rst b/doc/source/api.rst new file mode 100644 index 000000000..76632ff15 --- /dev/null +++ b/doc/source/api.rst @@ -0,0 +1,10 @@ +=========== +The Ray API +=========== + +.. autofunction:: ray.put +.. autofunction:: ray.get +.. autofunction:: ray.remote +.. autofunction:: ray.wait +.. autofunction:: ray.init +.. autofunction:: ray.error_info diff --git a/doc/conf.py b/doc/source/conf.py similarity index 96% rename from doc/conf.py rename to doc/source/conf.py index d6c124f39..c08c8ab8b 100644 --- a/doc/conf.py +++ b/doc/source/conf.py @@ -16,17 +16,16 @@ import sys import os import shlex -# These 4 lines added to enable ReadTheDocs to work. +# These lines added to enable Sphinx to work without installing Ray. import mock -MOCK_MODULES = ["numpy", "funcsigs", "colorama", "cloudpickle"] +MOCK_MODULES = ["global_scheduler", "numbuf", "local_scheduler", "plasma"] for mod_name in MOCK_MODULES: sys.modules[mod_name] = mock.Mock() # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath("../lib/python/")) -sys.path.insert(0, os.path.abspath("../scripts/")) +sys.path.insert(0, os.path.abspath("../../python/")) # -- General configuration ------------------------------------------------ @@ -38,7 +37,6 @@ sys.path.insert(0, os.path.abspath("../scripts/")) # ones. extensions = [ 'sphinx.ext.autodoc', - 'sphinx.ext.pngmath', 'sphinx.ext.napoleon', ] @@ -118,7 +116,9 @@ todo_include_todos = False # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'alabaster' +import sphinx_rtd_theme +html_theme = 'sphinx_rtd_theme' +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -163,7 +163,7 @@ html_static_path = ['_static'] #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +html_sidebars = {'**': ['index.html']} # Additional templates that should be rendered to pages, maps page names to # template names. diff --git a/examples/hyperopt/README.md b/doc/source/example-hyperopt.md similarity index 100% rename from examples/hyperopt/README.md rename to doc/source/example-hyperopt.md diff --git a/examples/lbfgs/README.md b/doc/source/example-lbfgs.md similarity index 100% rename from examples/lbfgs/README.md rename to doc/source/example-lbfgs.md diff --git a/examples/rl_pong/README.md b/doc/source/example-rl-pong.md similarity index 100% rename from examples/rl_pong/README.md rename to doc/source/example-rl-pong.md diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 000000000..7fbb9fffb --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,45 @@ +=== +Ray +=== + +*Ray is a low-latency distributed execution framework targeted at machine +learning and reinforcement learning applications.* + +.. toctree:: + :maxdepth: 0 + :caption: Installation + + install-on-ubuntu.md + install-on-macosx.md + install-on-docker.md + installation-troubleshooting.md + +.. toctree:: + :maxdepth: 0 + :caption: Examples + + example-hyperopt.md + example-lbfgs.md + example-rl-pong.md + using-ray-with-tensorflow.md + +.. toctree:: + :maxdepth: 0 + :caption: Getting Started + + api.rst + tutorial.md + +.. toctree:: + :maxdepth: 1 + :caption: Design + + remote-functions.md + serialization.md + +.. toctree:: + :maxdepth: 1 + :caption: Cluster Usage + + using-ray-on-a-cluster.md + using-ray-on-a-large-cluster.md diff --git a/doc/install-on-docker.md b/doc/source/install-on-docker.md similarity index 100% rename from doc/install-on-docker.md rename to doc/source/install-on-docker.md diff --git a/doc/install-on-macosx.md b/doc/source/install-on-macosx.md similarity index 98% rename from doc/install-on-macosx.md rename to doc/source/install-on-macosx.md index bef43a58d..235abac5a 100644 --- a/doc/install-on-macosx.md +++ b/doc/source/install-on-macosx.md @@ -20,7 +20,7 @@ If you are using Anaconda, you may also need to run the following. conda install libgcc ``` -# Install Ray +## Install Ray Ray can be built from the repository as follows. diff --git a/doc/install-on-ubuntu.md b/doc/source/install-on-ubuntu.md similarity index 99% rename from doc/install-on-ubuntu.md rename to doc/source/install-on-ubuntu.md index 7a1a3a899..c94fcdb02 100644 --- a/doc/install-on-ubuntu.md +++ b/doc/source/install-on-ubuntu.md @@ -21,7 +21,7 @@ If you are using Anaconda, you may also need to run the following. conda install libgcc ``` -# Install Ray +## Install Ray Ray can be built from the repository as follows. diff --git a/doc/installation-troubleshooting.md b/doc/source/installation-troubleshooting.md similarity index 100% rename from doc/installation-troubleshooting.md rename to doc/source/installation-troubleshooting.md diff --git a/doc/remote-functions.md b/doc/source/remote-functions.md similarity index 100% rename from doc/remote-functions.md rename to doc/source/remote-functions.md diff --git a/doc/serialization.md b/doc/source/serialization.md similarity index 99% rename from doc/serialization.md rename to doc/source/serialization.md index bd471763f..c9f0dbcd2 100644 --- a/doc/serialization.md +++ b/doc/source/serialization.md @@ -137,7 +137,7 @@ It will throw an exception with a message like the following. This object exceeds the maximum recursion depth. It may contain itself recursively. ``` -# Last Resort Workaround +## Last Resort Workaround If you find cases where Ray doesn't work or does the wrong thing, please let us know so we can fix it. In the meantime, you can do your own custom serialization diff --git a/doc/tutorial.md b/doc/source/tutorial.md similarity index 100% rename from doc/tutorial.md rename to doc/source/tutorial.md diff --git a/doc/using-ray-on-a-cluster.md b/doc/source/using-ray-on-a-cluster.md similarity index 100% rename from doc/using-ray-on-a-cluster.md rename to doc/source/using-ray-on-a-cluster.md diff --git a/doc/using-ray-on-a-large-cluster.md b/doc/source/using-ray-on-a-large-cluster.md similarity index 100% rename from doc/using-ray-on-a-large-cluster.md rename to doc/source/using-ray-on-a-large-cluster.md diff --git a/doc/using-ray-with-tensorflow.md b/doc/source/using-ray-with-tensorflow.md similarity index 100% rename from doc/using-ray-with-tensorflow.md rename to doc/source/using-ray-with-tensorflow.md diff --git a/python/ray/worker.py b/python/ray/worker.py index f43ae56bf..2369961c6 100644 --- a/python/ray/worker.py +++ b/python/ray/worker.py @@ -435,7 +435,7 @@ class Worker(object): Args: objectid (object_id.ObjectID): The object ID of the value to be put. - value (serializable object): The value to put in the object store. + value: The value to put in the object store. """ # Serialize and put the object in the object store. try: @@ -1455,7 +1455,7 @@ def put(value, worker=global_worker): """Store an object in the object store. Args: - value (serializable object): The Python object to be stored. + value: The Python object to be stored. Returns: The object ID assigned to this value. @@ -1485,8 +1485,8 @@ def wait(object_ids, num_returns=1, timeout=None, worker=global_worker): corresponds to the rest of the object IDs (which may or may not be ready). Args: - object_ids (List[ObjectID]): List of object IDs for objects that may - or may not be ready. Note that these IDs must be unique. + object_ids (List[ObjectID]): List of object IDs for objects that may or may + not be ready. Note that these IDs must be unique. num_returns (int): The number of object IDs that should be returned. timeout (int): The maximum amount of time in milliseconds to wait before returning. diff --git a/src/common/thirdparty/python b/src/common/thirdparty/python deleted file mode 160000 index 3f8fa0052..000000000 --- a/src/common/thirdparty/python +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3f8fa00528daa3e3849be251f05227842905c7a9 diff --git a/src/common/thirdparty/redis-windows b/src/common/thirdparty/redis-windows deleted file mode 160000 index 10a978f7b..000000000 --- a/src/common/thirdparty/redis-windows +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 10a978f7b4b52166724e7f00e5c9ac5a12aba3e1