mirror of
https://github.com/wassname/ray.git
synced 2026-09-09 11:32:43 +08:00
[docs] Second push of changes (#5391)
This commit is contained in:
Richard Liaw
+153
-29
@@ -11,49 +11,169 @@
|
||||
|
||||
|
|
||||
|
||||
|
||||
**Ray is a fast and simple framework for building and running distributed applications.**
|
||||
|
||||
Ray is packaged with the following libraries for accelerating machine learning workloads:
|
||||
|
||||
Ray is easy to install: ``pip install ray``
|
||||
- `Tune`_: Scalable Hyperparameter Tuning
|
||||
- `RLlib`_: Scalable Reinforcement Learning
|
||||
- `Distributed Training <https://ray.readthedocs.io/en/latest/distributed_training.html>`__
|
||||
|
||||
Example Use
|
||||
Install Ray with: ``pip install ray``. For nightly wheels, see the `Installation page <https://ray.readthedocs.io/en/latest/installation.html>`__.
|
||||
|
||||
Quick Start
|
||||
-----------
|
||||
|
||||
+------------------------------------------------+----------------------------------------------------+
|
||||
| **Basic Python** | **Distributed with Ray** |
|
||||
+------------------------------------------------+----------------------------------------------------+
|
||||
|.. code-block:: python |.. code-block:: python |
|
||||
| | |
|
||||
| # Execute f serially. | # Execute f in parallel. |
|
||||
| | |
|
||||
| | @ray.remote |
|
||||
| def f(): | def f(): |
|
||||
| time.sleep(1) | time.sleep(1) |
|
||||
| return 1 | return 1 |
|
||||
| | |
|
||||
| | |
|
||||
| | ray.init() |
|
||||
| results = [f() for i in range(4)] | results = ray.get([f.remote() for i in range(4)]) |
|
||||
+------------------------------------------------+----------------------------------------------------+
|
||||
Execute Python functions in parallel.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import ray
|
||||
ray.init()
|
||||
|
||||
@ray.remote
|
||||
def f(x):
|
||||
return x * x
|
||||
|
||||
futures = [f.remote(i) for i in range(4)]
|
||||
print(ray.get(futures))
|
||||
|
||||
To use Ray's actor model:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
|
||||
Ray comes with libraries that accelerate deep learning and reinforcement learning development:
|
||||
import ray
|
||||
ray.init()
|
||||
|
||||
- `Tune`_: Hyperparameter Optimization Framework
|
||||
- `RLlib`_: Scalable Reinforcement Learning
|
||||
- `Distributed Training <http://ray.readthedocs.io/en/latest/distributed_training.html>`__
|
||||
@ray.remote
|
||||
class Counter():
|
||||
def __init__(self):
|
||||
self.n = 0
|
||||
|
||||
.. _`Tune`: http://ray.readthedocs.io/en/latest/tune.html
|
||||
.. _`RLlib`: http://ray.readthedocs.io/en/latest/rllib.html
|
||||
def increment(self):
|
||||
self.n += 1
|
||||
|
||||
Installation
|
||||
------------
|
||||
def read(self):
|
||||
return self.n
|
||||
|
||||
Ray can be installed on Linux and Mac with ``pip install ray``.
|
||||
counters = [Counter.remote() for i in range(4)]
|
||||
[c.increment.remote() for c in counters]
|
||||
futures = [c.read.remote() for c in counters]
|
||||
print(ray.get(futures))
|
||||
|
||||
To build Ray from source or to install the nightly versions, see the `installation documentation`_.
|
||||
|
||||
.. _`installation documentation`: http://ray.readthedocs.io/en/latest/installation.html
|
||||
Ray programs can run on a single machine, and can also seamlessly scale to large clusters. To execute the above Ray script in the cloud, just download `this configuration file <https://github.com/ray-project/ray/blob/master/python/ray/autoscaler/aws/example-full.yaml>`__, and run:
|
||||
|
||||
``ray submit [CLUSTER.YAML] example.py --start``
|
||||
|
||||
Read more about `launching clusters <https://ray.readthedocs.io/en/latest/autoscaling.html>`_.
|
||||
|
||||
Tune Quick Start
|
||||
----------------
|
||||
|
||||
.. image:: https://github.com/ray-project/ray/raw/master/doc/source/images/tune-wide.png
|
||||
|
||||
`Tune`_ is a library for hyperparameter tuning at any scale.
|
||||
|
||||
- Launch a multi-node distributed hyperparameter sweep in less than 10 lines of code.
|
||||
- Supports any deep learning framework, including PyTorch, TensorFlow, and Keras.
|
||||
- Visualize results with `TensorBoard <https://www.tensorflow.org/get_started/summaries_and_tensorboard>`__.
|
||||
- Choose among scalable SOTA algorithms such as `Population Based Training (PBT)`_, `Vizier's Median Stopping Rule`_, `HyperBand/ASHA`_.
|
||||
- Tune integrates with many optimization libraries such as `Facebook Ax <http://ax.dev>`_, `HyperOpt <https://github.com/hyperopt/hyperopt>`_, and `Bayesian Optimization <https://github.com/fmfn/BayesianOptimization>`_ and enables you to scale them transparently.
|
||||
|
||||
To run this example, you will need to install the following:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ pip install ray torch torchvision filelock
|
||||
|
||||
|
||||
This example runs a parallel grid search to train a Convolutional Neural Network using PyTorch.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
|
||||
import torch.optim as optim
|
||||
from ray import tune
|
||||
from ray.tune.examples.mnist_pytorch import (
|
||||
get_data_loaders, ConvNet, train, test)
|
||||
|
||||
|
||||
def train_mnist(config):
|
||||
train_loader, test_loader = get_data_loaders()
|
||||
model = ConvNet()
|
||||
optimizer = optim.SGD(model.parameters(), lr=config["lr"])
|
||||
for i in range(10):
|
||||
train(model, optimizer, train_loader)
|
||||
acc = test(model, test_loader)
|
||||
tune.track.log(mean_accuracy=acc)
|
||||
|
||||
|
||||
analysis = tune.run(
|
||||
train_mnist, config={"lr": tune.grid_search([0.001, 0.01, 0.1])})
|
||||
|
||||
print("Best config: ", analysis.get_best_config(metric="mean_accuracy"))
|
||||
|
||||
# Get a dataframe for analyzing trial results.
|
||||
df = analysis.dataframe()
|
||||
|
||||
If TensorBoard is installed, automatically visualize all trial results:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
tensorboard --logdir ~/ray_results
|
||||
|
||||
.. _`Tune`: https://ray.readthedocs.io/en/latest/tune.html
|
||||
.. _`Population Based Training (PBT)`: https://ray.readthedocs.io/en/latest/tune-schedulers.html#population-based-training-pbt
|
||||
.. _`Vizier's Median Stopping Rule`: https://ray.readthedocs.io/en/latest/tune-schedulers.html#median-stopping-rule
|
||||
.. _`HyperBand/ASHA`: https://ray.readthedocs.io/en/latest/tune-schedulers.html#asynchronous-hyperband
|
||||
|
||||
RLlib Quick Start
|
||||
-----------------
|
||||
|
||||
`RLlib`_ is an open-source library for reinforcement learning built on top of Ray that offers both high scalability and a unified API for a variety of applications.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip install tensorflow # or tensorflow-gpu
|
||||
pip install ray[rllib] # also recommended: ray[debug]
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import gym
|
||||
from gym.spaces import Discrete, Box
|
||||
from ray import tune
|
||||
|
||||
class SimpleCorridor(gym.Env):
|
||||
def __init__(self, config):
|
||||
self.end_pos = config["corridor_length"]
|
||||
self.cur_pos = 0
|
||||
self.action_space = Discrete(2)
|
||||
self.observation_space = Box(0.0, self.end_pos, shape=(1, ))
|
||||
|
||||
def reset(self):
|
||||
self.cur_pos = 0
|
||||
return [self.cur_pos]
|
||||
|
||||
def step(self, action):
|
||||
if action == 0 and self.cur_pos > 0:
|
||||
self.cur_pos -= 1
|
||||
elif action == 1:
|
||||
self.cur_pos += 1
|
||||
done = self.cur_pos >= self.end_pos
|
||||
return [self.cur_pos], 1 if done else 0, done, {}
|
||||
|
||||
tune.run(
|
||||
"PPO",
|
||||
config={
|
||||
"env": SimpleCorridor,
|
||||
"num_workers": 4,
|
||||
"env_config": {"corridor_length": 5}})
|
||||
|
||||
.. _`RLlib`: https://ray.readthedocs.io/en/latest/rllib.html
|
||||
|
||||
|
||||
More Information
|
||||
----------------
|
||||
@@ -63,12 +183,16 @@ More Information
|
||||
- `Blog`_
|
||||
- `Ray paper`_
|
||||
- `Ray HotOS paper`_
|
||||
- `RLlib paper`_
|
||||
- `Tune paper`_
|
||||
|
||||
.. _`Documentation`: http://ray.readthedocs.io/en/latest/index.html
|
||||
.. _`Tutorial`: https://github.com/ray-project/tutorial
|
||||
.. _`Blog`: https://ray-project.github.io/
|
||||
.. _`Ray paper`: https://arxiv.org/abs/1712.05889
|
||||
.. _`Ray HotOS paper`: https://arxiv.org/abs/1703.03924
|
||||
.. _`RLlib paper`: https://arxiv.org/abs/1712.09381
|
||||
.. _`Tune paper`: https://arxiv.org/abs/1807.05118
|
||||
|
||||
Getting Involved
|
||||
----------------
|
||||
|
||||
@@ -40,4 +40,16 @@ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE}
|
||||
python /ray/python/ray/experimental/sgd/examples/train_example.py --num-replicas=2
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
|
||||
python /ray/python/ray/experimental/sgd/examples/train_example.py --tune
|
||||
python /ray/python/ray/experimental/sgd/examples/tune_example.py
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
|
||||
python /ray/python/ray/experimental/sgd/examples/tune_example.py
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
|
||||
python /ray/python/ray/experimental/sgd/examples/tune_example.py --num-replicas=2
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
|
||||
python /ray/doc/examples/doc_code/torch_example.py
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
|
||||
python /ray/doc/examples/doc_code/tf_example.py
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
# flake8: noqa
|
||||
"""
|
||||
This file holds code for the TF best-practices guide in the documentation.
|
||||
|
||||
It ignores yapf because yapf doesn't allow comments right after code blocks,
|
||||
but we put comments right after code blocks to prevent large white spaces
|
||||
in the documentation.
|
||||
"""
|
||||
|
||||
# yapf: disable
|
||||
# __tf_model_start__
|
||||
import tensorflow as tf
|
||||
from tensorflow.keras import layers
|
||||
|
||||
|
||||
def create_keras_model():
|
||||
model = tf.keras.Sequential()
|
||||
# Adds a densely-connected layer with 64 units to the model:
|
||||
model.add(layers.Dense(64, activation="relu", input_shape=(32, )))
|
||||
# Add another:
|
||||
model.add(layers.Dense(64, activation="relu"))
|
||||
# Add a softmax layer with 10 output units:
|
||||
model.add(layers.Dense(10, activation="softmax"))
|
||||
|
||||
model.compile(
|
||||
optimizer=tf.train.RMSPropOptimizer(0.01),
|
||||
loss=tf.keras.losses.categorical_crossentropy,
|
||||
metrics=[tf.keras.metrics.categorical_accuracy])
|
||||
return model
|
||||
# __tf_model_end__
|
||||
# yapf: enable
|
||||
|
||||
# yapf: disable
|
||||
# __ray_start__
|
||||
import ray
|
||||
import numpy as np
|
||||
|
||||
ray.init()
|
||||
|
||||
def random_one_hot_labels(shape):
|
||||
n, n_class = shape
|
||||
classes = np.random.randint(0, n_class, n)
|
||||
labels = np.zeros((n, n_class))
|
||||
labels[np.arange(n), classes] = 1
|
||||
return labels
|
||||
|
||||
|
||||
# Use GPU wth
|
||||
# @ray.remote(num_gpus=1)
|
||||
@ray.remote
|
||||
class Network():
|
||||
def __init__(self):
|
||||
self.model = create_keras_model()
|
||||
self.dataset = np.random.random((1000, 32))
|
||||
self.labels = random_one_hot_labels((1000, 10))
|
||||
|
||||
def train(self):
|
||||
history = self.model.fit(self.dataset, self.labels, verbose=False)
|
||||
return history.history
|
||||
|
||||
def get_weights(self):
|
||||
return self.model.get_weights()
|
||||
|
||||
def set_weights(self, weights):
|
||||
# Note that for simplicity this does not handle the optimizer state.
|
||||
self.model.set_weights(weights)
|
||||
# __ray_end__
|
||||
# yapf: enable
|
||||
|
||||
# yapf: disable
|
||||
# __actor_start__
|
||||
NetworkActor = Network.remote()
|
||||
result_object_id = NetworkActor.train.remote()
|
||||
ray.get(result_object_id)
|
||||
# __actor_end__
|
||||
# yapf: enable
|
||||
|
||||
# yapf: disable
|
||||
# __weight_average_start__
|
||||
NetworkActor2 = Network.remote()
|
||||
NetworkActor2.train.remote()
|
||||
weights = ray.get(
|
||||
[NetworkActor.get_weights.remote(),
|
||||
NetworkActor2.get_weights.remote()])
|
||||
|
||||
averaged_weights = [(layer1 + layer2) / 2
|
||||
for layer1, layer2 in zip(weights[0], weights[1])]
|
||||
|
||||
weight_id = ray.put(averaged_weights)
|
||||
[
|
||||
actor.set_weights.remote(weight_id)
|
||||
for actor in [NetworkActor, NetworkActor2]
|
||||
]
|
||||
ray.get([actor.train.remote() for actor in [NetworkActor, NetworkActor2]])
|
||||
@@ -0,0 +1,180 @@
|
||||
# flake8: noqa
|
||||
"""
|
||||
This file holds code for the Torch best-practices guide in the documentation.
|
||||
|
||||
It ignores yapf because yapf doesn't allow comments right after code blocks,
|
||||
but we put comments right after code blocks to prevent large white spaces
|
||||
in the documentation.
|
||||
"""
|
||||
# yapf: disable
|
||||
# __torch_model_start__
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
class Model(nn.Module):
|
||||
def __init__(self):
|
||||
super(Model, self).__init__()
|
||||
self.conv1 = nn.Conv2d(1, 20, 5, 1)
|
||||
self.conv2 = nn.Conv2d(20, 50, 5, 1)
|
||||
self.fc1 = nn.Linear(4 * 4 * 50, 500)
|
||||
self.fc2 = nn.Linear(500, 10)
|
||||
|
||||
def forward(self, x):
|
||||
x = F.relu(self.conv1(x))
|
||||
x = F.max_pool2d(x, 2, 2)
|
||||
x = F.relu(self.conv2(x))
|
||||
x = F.max_pool2d(x, 2, 2)
|
||||
x = x.view(-1, 4 * 4 * 50)
|
||||
x = F.relu(self.fc1(x))
|
||||
x = self.fc2(x)
|
||||
return F.log_softmax(x, dim=1)
|
||||
# __torch_model_end__
|
||||
# yapf: enable
|
||||
|
||||
# yapf: disable
|
||||
# __torch_helper_start__
|
||||
from filelock import FileLock
|
||||
from torchvision import datasets, transforms
|
||||
|
||||
|
||||
def train(model, device, train_loader, optimizer):
|
||||
model.train()
|
||||
for batch_idx, (data, target) in enumerate(train_loader):
|
||||
# This break is for speeding up the tutorial.
|
||||
if batch_idx * len(data) > 1024:
|
||||
return
|
||||
data, target = data.to(device), target.to(device)
|
||||
optimizer.zero_grad()
|
||||
output = model(data)
|
||||
loss = F.nll_loss(output, target)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
|
||||
def test(model, device, test_loader):
|
||||
model.eval()
|
||||
test_loss = 0
|
||||
correct = 0
|
||||
with torch.no_grad():
|
||||
for data, target in test_loader:
|
||||
data, target = data.to(device), target.to(device)
|
||||
output = model(data)
|
||||
|
||||
# sum up batch loss
|
||||
test_loss += F.nll_loss(
|
||||
output, target, reduction="sum").item()
|
||||
pred = output.argmax(
|
||||
dim=1,
|
||||
keepdim=True)
|
||||
correct += pred.eq(target.view_as(pred)).sum().item()
|
||||
|
||||
test_loss /= len(test_loader.dataset)
|
||||
return {
|
||||
"loss": test_loss,
|
||||
"accuracy": 100. * correct / len(test_loader.dataset)
|
||||
}
|
||||
|
||||
|
||||
def dataset_creators(use_cuda):
|
||||
kwargs = {"num_workers": 1, "pin_memory": True} if use_cuda else {}
|
||||
with FileLock("./data.lock"):
|
||||
train_loader = torch.utils.data.DataLoader(
|
||||
datasets.MNIST(
|
||||
"./data",
|
||||
train=True,
|
||||
download=True,
|
||||
transform=transforms.Compose([
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize((0.1307, ), (0.3081, ))
|
||||
])),
|
||||
128,
|
||||
shuffle=True,
|
||||
**kwargs)
|
||||
test_loader = torch.utils.data.DataLoader(
|
||||
datasets.MNIST(
|
||||
"./data",
|
||||
train=False,
|
||||
transform=transforms.Compose([
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize((0.1307, ), (0.3081, ))
|
||||
])),
|
||||
128,
|
||||
shuffle=True,
|
||||
**kwargs)
|
||||
|
||||
return train_loader, test_loader
|
||||
# __torch_helper_end__
|
||||
# yapf: enable
|
||||
|
||||
# yapf: disable
|
||||
# __torch_net_start__
|
||||
import torch.optim as optim
|
||||
|
||||
|
||||
class Network():
|
||||
def __init__(self, lr=0.01, momentum=0.5):
|
||||
use_cuda = torch.cuda.is_available()
|
||||
self.device = device = torch.device("cuda" if use_cuda else "cpu")
|
||||
self.train_loader, self.test_loader = dataset_creators(use_cuda)
|
||||
|
||||
self.model = Model().to(device)
|
||||
self.optimizer = optim.SGD(
|
||||
self.model.parameters(), lr=lr, momentum=momentum)
|
||||
|
||||
def train(self):
|
||||
train(self.model, self.device, self.train_loader, self.optimizer)
|
||||
return test(self.model, self.device, self.test_loader)
|
||||
|
||||
def get_weights(self):
|
||||
return self.model.state_dict()
|
||||
|
||||
def set_weights(self, weights):
|
||||
self.model.load_state_dict(weights)
|
||||
|
||||
def save(self):
|
||||
torch.save(self.model.state_dict(), "mnist_cnn.pt")
|
||||
|
||||
|
||||
net = Network()
|
||||
net.train()
|
||||
# __torch_net_end__
|
||||
# yapf: enable
|
||||
|
||||
# yapf: disable
|
||||
# __torch_ray_start__
|
||||
import ray
|
||||
ray.init()
|
||||
|
||||
RemoteNetwork = ray.remote(Network)
|
||||
# Use the below instead of `ray.remote(network)` to leverage the GPU.
|
||||
# RemoteNetwork = ray.remote(num_gpus=1)(Network)
|
||||
# __torch_ray_end__
|
||||
# yapf: enable
|
||||
|
||||
# yapf: disable
|
||||
# __torch_actor_start__
|
||||
NetworkActor = RemoteNetwork.remote()
|
||||
NetworkActor2 = RemoteNetwork.remote()
|
||||
|
||||
ray.get([NetworkActor.train.remote(), NetworkActor2.train.remote()])
|
||||
# __torch_actor_end__
|
||||
# yapf: enable
|
||||
|
||||
# yapf: disable
|
||||
# __weight_average_start__
|
||||
weights = ray.get(
|
||||
[NetworkActor.get_weights.remote(),
|
||||
NetworkActor2.get_weights.remote()])
|
||||
|
||||
from collections import OrderedDict
|
||||
averaged_weights = OrderedDict(
|
||||
[(k, (weights[0][k] + weights[1][k]) / 2) for k in weights[0]])
|
||||
|
||||
weight_id = ray.put(averaged_weights)
|
||||
[
|
||||
actor.set_weights.remote(weight_id)
|
||||
for actor in [NetworkActor, NetworkActor2]
|
||||
]
|
||||
ray.get([actor.train.remote() for actor in [NetworkActor, NetworkActor2]])
|
||||
+21
-6
@@ -42,6 +42,9 @@ When the above actor is instantiated, the following events happen.
|
||||
2. A ``Counter`` object is created on that worker and the ``Counter``
|
||||
constructor is run.
|
||||
|
||||
Actor Methods
|
||||
-------------
|
||||
|
||||
Any method of the actor can return multiple object IDs with the ``ray.method`` decorator:
|
||||
|
||||
.. code-block:: python
|
||||
@@ -59,6 +62,7 @@ Any method of the actor can return multiple object IDs with the ``ray.method`` d
|
||||
assert ray.get(obj_id1) == 1
|
||||
assert ray.get(obj_id2) == 2
|
||||
|
||||
|
||||
Resources with Actors
|
||||
---------------------
|
||||
|
||||
@@ -95,11 +99,28 @@ have these resources (see `configuration instructions
|
||||
lifetime, but every time it executes a method, it will need to acquire 1 CPU
|
||||
resource.
|
||||
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@ray.remote(resources={'Resource2': 1})
|
||||
class GPUActor(object):
|
||||
pass
|
||||
|
||||
|
||||
If you need to instantiate many copies of the same actor with varying resource
|
||||
requirements, you can do so as follows.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@ray.remote(num_cpus=4)
|
||||
class Counter(object):
|
||||
def __init__(self):
|
||||
self.value = 0
|
||||
|
||||
def increment(self):
|
||||
self.value += 1
|
||||
return self.value
|
||||
|
||||
a1 = Counter._remote(num_cpus=1, resources={"Custom1": 1})
|
||||
a2 = Counter._remote(num_cpus=2, resources={"Custom2": 1})
|
||||
a3 = Counter._remote(num_cpus=3, resources={"Custom3": 1})
|
||||
@@ -107,12 +128,6 @@ requirements, you can do so as follows.
|
||||
Note that to create these actors successfully, Ray will need to be started with
|
||||
sufficient CPU resources and the relevant custom resources.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@ray.remote(resources={'Resource2': 1})
|
||||
class GPUActor(object):
|
||||
pass
|
||||
|
||||
|
||||
Terminating Actors
|
||||
------------------
|
||||
|
||||
+49
-111
@@ -3,6 +3,55 @@ Advanced Usage
|
||||
|
||||
This page will cover some more advanced examples of using Ray's flexible programming model.
|
||||
|
||||
Dynamic Remote Parameters
|
||||
-------------------------
|
||||
|
||||
You can dynamically adjust resource requirements or return values of ``ray.remote`` during execution with ``._remote``.
|
||||
|
||||
For example, here we instantiate many copies of the same actor with varying resource requirements. Note that to create these actors successfully, Ray will need to be started with sufficient CPU resources and the relevant custom resources:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@ray.remote(num_cpus=4)
|
||||
class Counter(object):
|
||||
def __init__(self):
|
||||
self.value = 0
|
||||
|
||||
def increment(self):
|
||||
self.value += 1
|
||||
return self.value
|
||||
|
||||
a1 = Counter._remote(num_cpus=1, resources={"Custom1": 1})
|
||||
a2 = Counter._remote(num_cpus=2, resources={"Custom2": 1})
|
||||
a3 = Counter._remote(num_cpus=3, resources={"Custom3": 1})
|
||||
|
||||
You can specify different resource requirements for tasks (but not for actor methods):
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@ray.remote
|
||||
def g():
|
||||
return ray.get_gpu_ids()
|
||||
|
||||
object_gpu_ids = g.remote()
|
||||
assert ray.get(object_gpu_ids) == [0]
|
||||
|
||||
dynamic_object_gpu_ids = g._remote(args=[], num_cpus=1, num_gpus=1)
|
||||
assert ray.get(dynamic_object_gpu_ids) == [0]
|
||||
|
||||
And vary the number of return values for tasks (and actor methods too):
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@ray.remote
|
||||
def f(n):
|
||||
return list(range(n))
|
||||
|
||||
id1, id2 = f._remote(args=[2], num_return_vals=2)
|
||||
assert ray.get(id1) == 0
|
||||
assert ray.get(id2) == 1
|
||||
|
||||
|
||||
Nested Remote Functions
|
||||
-----------------------
|
||||
|
||||
@@ -94,114 +143,3 @@ Notes
|
||||
* Several limitations come from Cython's own `unsupported <https://github.com/cython/cython/wiki/Unsupported>`_ Python features.
|
||||
* We currently do not support compiling and distributing Cython code to ``ray`` clusters. In other words, Cython developers are responsible for compiling and distributing any Cython code to their cluster (much as would be the case for users who need Python packages like ``scipy``).
|
||||
* For most simple use cases, developers need not worry about Python 2 or 3, but users who do need to care can have a look at the ``language_level`` Cython compiler directive (see `here <http://cython.readthedocs.io/en/latest/src/reference/compilation.html>`_).
|
||||
|
||||
Serialization
|
||||
-------------
|
||||
|
||||
There are a number of situations in which Ray will place objects in the object
|
||||
store. Once an object is placed in the object store, it is immutable. Situations include:
|
||||
|
||||
1. The return values of a remote function.
|
||||
2. The value ``x`` in a call to ``ray.put(x)``.
|
||||
3. Arguments to remote functions (except for simple arguments like ints or
|
||||
floats).
|
||||
|
||||
A Python object may have an arbitrary number of pointers with arbitrarily deep
|
||||
nesting. To place an object in the object store or send it between processes,
|
||||
it must first be converted to a contiguous string of bytes. Serialization and deserialization can often be a bottleneck.
|
||||
|
||||
Pickle is standard Python serialization library. However, for numerical workloads, pickling and unpickling can be inefficient. For example, if multiple processes want to access a Python list of numpy arrays, each process must unpickle the list and create its own new copies of the arrays. This can lead to high memory overheads, even when all processes are read-only and could easily share memory.
|
||||
|
||||
In Ray, we optimize for numpy arrays by using the `Apache Arrow`_ data format.
|
||||
When we deserialize a list of numpy arrays from the object store, we still
|
||||
create a Python list of numpy array objects. However, rather than copy each
|
||||
numpy array, each numpy array object holds a pointer to the relevant array held
|
||||
in shared memory. There are some advantages to this form of serialization.
|
||||
|
||||
- Deserialization can be very fast.
|
||||
- Memory is shared between processes so worker processes can all read the same
|
||||
data without having to copy it.
|
||||
|
||||
.. _`Apache Arrow`: https://arrow.apache.org/
|
||||
|
||||
What Objects Does Ray Handle
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Ray does not currently support serialization of arbitrary Python objects. The
|
||||
set of Python objects that Ray can serialize using Arrow includes the following.
|
||||
|
||||
1. Primitive types: ints, floats, longs, bools, strings, unicode, and numpy
|
||||
arrays.
|
||||
2. Any list, dictionary, or tuple whose elements can be serialized by Ray.
|
||||
|
||||
For a more general object, Ray will first attempt to serialize the object by
|
||||
unpacking the object as a dictionary of its fields. This behavior is not
|
||||
correct in all cases. If Ray cannot serialize the object as a dictionary of its
|
||||
fields, Ray will fall back to using pickle. However, using pickle will likely
|
||||
be inefficient.
|
||||
|
||||
Notes and limitations
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- We currently handle certain patterns incorrectly, according to Python
|
||||
semantics. For example, a list that contains two copies of the same list will
|
||||
be serialized as if the two lists were distinct.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
l1 = [0]
|
||||
l2 = [l1, l1]
|
||||
l3 = ray.get(ray.put(l2))
|
||||
|
||||
l2[0] is l2[1] # True.
|
||||
l3[0] is l3[1] # False.
|
||||
|
||||
- For reasons similar to the above example, we also do not currently handle
|
||||
objects that recursively contain themselves (this may be common in graph-like
|
||||
data structures).
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
l = []
|
||||
l.append(l)
|
||||
|
||||
# Try to put this list that recursively contains itself in the object store.
|
||||
ray.put(l)
|
||||
|
||||
This will throw an exception with a message like the following.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
This object exceeds the maximum recursion depth. It may contain itself recursively.
|
||||
|
||||
- Whenever possible, use numpy arrays for maximum performance.
|
||||
|
||||
Last Resort Workaround
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
If you find cases where Ray serialization doesn't work or does something
|
||||
unexpected, please `let us know`_ so we can fix it. In the meantime, you may
|
||||
have to resort to writing custom serialization and deserialization code (e.g.,
|
||||
calling pickle by hand).
|
||||
|
||||
.. _`let us know`: https://github.com/ray-project/ray/issues
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import pickle
|
||||
|
||||
@ray.remote
|
||||
def f(complicated_object):
|
||||
# Deserialize the object manually.
|
||||
obj = pickle.loads(complicated_object)
|
||||
return "Successfully passed {} into f.".format(obj)
|
||||
|
||||
# Define a complicated object.
|
||||
l = []
|
||||
l.append(l)
|
||||
|
||||
# Manually serialize the object and pass it in as a string.
|
||||
ray.get(f.remote(pickle.dumps(l))) # prints 'Successfully passed [[...]] into f.'
|
||||
|
||||
**Note:** If you have trouble with pickle, you may have better luck with
|
||||
cloudpickle.
|
||||
|
||||
@@ -5,15 +5,6 @@ Since Python 3.5, it is possible to write concurrent code using the ``async/awai
|
||||
|
||||
This document describes Ray's support for asyncio, which enables integration with popular async frameworks (e.g., aiohttp, aioredis, etc.) for high performance web and prediction serving.
|
||||
|
||||
Starting Ray
|
||||
------------
|
||||
|
||||
You must initialize Ray first.
|
||||
|
||||
Please refer to `Starting Ray`_ for instructions.
|
||||
|
||||
.. _`Starting Ray`: http://ray.readthedocs.io/en/latest/tutorial.html#starting-ray
|
||||
|
||||
|
||||
Converting Ray objects into asyncio futures
|
||||
-------------------------------------------
|
||||
|
||||
@@ -5,6 +5,9 @@ This page discusses the various way to configure Ray, both from the Python API
|
||||
and from the command line. Take a look at the ``ray.init`` `documentation
|
||||
<package-ref.html#ray.init>`__ for a complete overview of the configurations.
|
||||
|
||||
.. important:: For the multi-node setting, you must first run `ray start` on the command line before ``ray.init`` in Python. On a single machine, you can run ``ray.init()`` without `ray start`.
|
||||
|
||||
|
||||
Cluster Resources
|
||||
-----------------
|
||||
|
||||
@@ -143,7 +146,6 @@ Plasma is a high-performance shared memory object store originally developed in
|
||||
Ray and now being developed in `Apache Arrow`_. See the `relevant
|
||||
documentation`_.
|
||||
|
||||
|
||||
On Linux, it is possible to increase the write throughput of the Plasma object
|
||||
store by using huge pages. You first need to create a file system and activate
|
||||
huge pages as follows.
|
||||
|
||||
@@ -79,7 +79,7 @@ following.
|
||||
|
||||
One of the pods will download and run `this example script`_.
|
||||
|
||||
.. _`this example script`: https://github.com/ray-project/ray/blob/master/doc/kubernetes/example.py
|
||||
.. _`this example script`: https://github.com/ray-project/ray/tree/master/doc/kubernetes/example.py
|
||||
|
||||
The script prints its output. To view the output, first find the pod name by
|
||||
running ``kubectl get all``. You'll see output like the following.
|
||||
|
||||
@@ -1,48 +1,106 @@
|
||||
Distributed Training (Experimental)
|
||||
===================================
|
||||
|
||||
Ray's ``PyTorchTrainer`` simplifies distributed model training for PyTorch. The ``PyTorchTrainer`` is a wrapper around ``torch.distributed.launch`` with a Python API to easily incorporate distributed training into a larger Python application, as opposed to needing to execute training outside of Python.
|
||||
|
||||
Ray includes abstractions for distributed model training that integrate with
|
||||
deep learning frameworks, such as PyTorch.
|
||||
----------
|
||||
|
||||
Ray Train is built on top of the Ray task and actor abstractions to provide
|
||||
seamless integration into existing Ray applications.
|
||||
**With Ray**:
|
||||
|
||||
PyTorch Interface
|
||||
-----------------
|
||||
|
||||
To use Ray Train with PyTorch, pass model and data creator functions to the
|
||||
``ray.experimental.sgd.pytorch.PyTorchTrainer`` class.
|
||||
To drive the distributed training, ``trainer.train()`` can be called
|
||||
repeatedly.
|
||||
Wrap your training with this:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
model_creator = lambda config: YourPyTorchModel()
|
||||
data_creator = lambda config: YourTrainingSet(), YourValidationSet()
|
||||
|
||||
trainer = PyTorchTrainer(
|
||||
ray.init(args.address)
|
||||
trainer1 = PyTorchTrainer(
|
||||
model_creator,
|
||||
data_creator,
|
||||
optimizer_creator=utils.sgd_mse_optimizer,
|
||||
config={"lr": 1e-4},
|
||||
num_replicas=2,
|
||||
resources_per_replica=Resources(num_gpus=1),
|
||||
batch_size=16,
|
||||
backend="auto")
|
||||
optimizer_creator,
|
||||
num_replicas=<NUM_GPUS_YOU_HAVE> * <NUM_NODES>,
|
||||
use_gpu=True,
|
||||
batch_size=512,
|
||||
backend="gloo")
|
||||
|
||||
for i in range(NUM_EPOCHS):
|
||||
trainer.train()
|
||||
trainer1.train()
|
||||
|
||||
Under the hood, Ray Train will create *replicas* of your model
|
||||
(controlled by ``num_replicas``) which are each managed by a worker.
|
||||
Multiple devices (e.g. GPUs) can be managed by each replica (controlled by ``resources_per_replica``),
|
||||
which allows training of lage models across multiple GPUs.
|
||||
The ``PyTorchTrainer`` class coordinates the distributed computation and training to improve the model.
|
||||
|
||||
The full documentation for ``PyTorchTrainer`` is as follows:
|
||||
Then, start a Ray cluster `via autoscaler <autoscaling.html>`_ or `manually <using-ray-on-a-cluster.html>`_.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
ray up CLUSTER.yaml
|
||||
python train.py --address="localhost:<PORT>"
|
||||
|
||||
|
||||
----------
|
||||
|
||||
**Before, with Pytorch**:
|
||||
|
||||
In your training program, insert the following:
|
||||
|
||||
.. code-block::
|
||||
|
||||
torch.distributed.init_process_group(backend='YOUR BACKEND',
|
||||
init_method='env://')
|
||||
|
||||
model = torch.nn.parallel.DistributedDataParallel(model,
|
||||
device_ids=[arg.local_rank],
|
||||
output_device=arg.local_rank)
|
||||
|
||||
Then, separately, on each machine:
|
||||
|
||||
.. code-block::
|
||||
|
||||
# Node 1: *(IP: 192.168.1.1, and has a free port: 1234)*
|
||||
$ python -m torch.distributed.launch --nproc_per_node=NUM_GPUS_YOU_HAVE
|
||||
--nnodes=4 --node_rank=0 --master_addr="192.168.1.1"
|
||||
--master_port=1234 YOUR_TRAINING_SCRIPT.py (--arg1 --arg2 --arg3
|
||||
and all other arguments of your training script)
|
||||
# Node 2:
|
||||
$ python -m torch.distributed.launch --nproc_per_node=NUM_GPUS_YOU_HAVE
|
||||
--nnodes=4 --node_rank=1 --master_addr="192.168.1.1"
|
||||
--master_port=1234 YOUR_TRAINING_SCRIPT.py (--arg1 --arg2 --arg3
|
||||
and all other arguments of your training script)
|
||||
# Node 3:
|
||||
$ python -m torch.distributed.launch --nproc_per_node=NUM_GPUS_YOU_HAVE
|
||||
--nnodes=4 --node_rank=2 --master_addr="192.168.1.1"
|
||||
--master_port=1234 YOUR_TRAINING_SCRIPT.py (--arg1 --arg2 --arg3
|
||||
and all other arguments of your training script)
|
||||
# Node 4:
|
||||
$ python -m torch.distributed.launch --nproc_per_node=NUM_GPUS_YOU_HAVE
|
||||
--nnodes=4 --node_rank=3 --master_addr="192.168.1.1"
|
||||
--master_port=1234 YOUR_TRAINING_SCRIPT.py (--arg1 --arg2 --arg3
|
||||
and all other arguments of your training script)
|
||||
|
||||
|
||||
PyTorchTrainer Example
|
||||
----------------------
|
||||
|
||||
Below is an example of using Ray's PyTorchTrainer. Under the hood, ``PytorchTrainer`` will create *replicas* of your model (controlled by ``num_replicas``) which are each managed by a worker.
|
||||
|
||||
.. literalinclude:: ../../python/ray/experimental/sgd/examples/train_example.py
|
||||
:language: python
|
||||
:start-after: __torch_train_example__
|
||||
|
||||
|
||||
Hyperparameter Optimization on Distributed Pytorch
|
||||
--------------------------------------------------
|
||||
|
||||
``PyTorchTrainer`` naturally integrates with Tune via the ``PyTorchTrainable`` interface. The same arguments to ``PyTorchTrainer`` should be passed into the ``tune.run(config=...)`` as shown below.
|
||||
|
||||
.. literalinclude:: ../../python/ray/experimental/sgd/examples/tune_example.py
|
||||
:language: python
|
||||
:start-after: __torch_tune_example__
|
||||
|
||||
|
||||
Package Reference
|
||||
-----------------
|
||||
|
||||
.. autoclass:: ray.experimental.sgd.pytorch.PyTorchTrainer
|
||||
:members:
|
||||
|
||||
.. automethod:: __init__
|
||||
|
||||
|
||||
.. autoclass:: ray.experimental.sgd.pytorch.PyTorchTrainable
|
||||
:members:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
ResNet
|
||||
======
|
||||
|
||||
This code adapts the `TensorFlow ResNet example`_ to do data parallel training
|
||||
This code uses ResNet to do data parallel training
|
||||
across multiple GPUs using Ray. View the `code for this example`_.
|
||||
|
||||
To run the example, you will need to install `TensorFlow`_ (at
|
||||
@@ -99,6 +99,5 @@ object store.
|
||||
mean_weights = {k: sum([weights[k] for weights in all_weights]) / num_gpus for k in all_weights[0]}
|
||||
weight_id = ray.put(mean_weights)
|
||||
|
||||
.. _`TensorFlow ResNet example`: https://github.com/tensorflow/models/tree/master/resnet
|
||||
.. _`TensorFlow`: https://www.tensorflow.org/install/
|
||||
.. _`code for this example`: https://github.com/ray-project/ray/tree/master/doc/examples/resnet
|
||||
|
||||
@@ -37,7 +37,7 @@ Process Failures
|
||||
~~~~~~~~~~~~~~~~
|
||||
|
||||
1. Ray does not recover from the failure of any of the following processes:
|
||||
a Redis server and the monitor process.
|
||||
any of the Redis servers and the monitor process.
|
||||
2. If a driver fails, that driver will not be restarted and the job will not
|
||||
complete.
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 44 KiB |
+49
-21
@@ -7,11 +7,13 @@ Ray
|
||||
<a href="https://github.com/ray-project/ray"><img style="position: absolute; top: 0; right: 0; border: 0;" src="https://camo.githubusercontent.com/365986a132ccd6a44c23a9169022c0b5c890c387/68747470733a2f2f73332e616d617a6f6e6177732e636f6d2f6769746875622f726962626f6e732f666f726b6d655f72696768745f7265645f6161303030302e706e67" alt="Fork me on GitHub" data-canonical-src="https://s3.amazonaws.com/github/ribbons/forkme_right_red_aa0000.png"></a>
|
||||
</embed>
|
||||
|
||||
*Ray is a fast and simple framework for building and running distributed applications.*
|
||||
.. image:: https://github.com/ray-project/ray/raw/master/doc/source/images/ray_header_logo.png
|
||||
|
||||
Ray comes with libraries that accelerate deep learning and reinforcement learning development:
|
||||
**Ray is a fast and simple framework for building and running distributed applications.**
|
||||
|
||||
- `Tune`_: Scalable Hyperparameter Search
|
||||
Ray is packaged with the following libraries for accelerating machine learning workloads:
|
||||
|
||||
- `Tune`_: Scalable Hyperparameter Tuning
|
||||
- `RLlib`_: Scalable Reinforcement Learning
|
||||
- `Distributed Training <distributed_training.html>`__
|
||||
|
||||
@@ -25,8 +27,11 @@ View the `codebase on GitHub`_.
|
||||
Quick Start
|
||||
-----------
|
||||
|
||||
Execute Python functions in parallel.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import ray
|
||||
ray.init()
|
||||
|
||||
@ray.remote
|
||||
@@ -40,6 +45,7 @@ To use Ray's actor model:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import ray
|
||||
ray.init()
|
||||
|
||||
@ray.remote
|
||||
@@ -58,17 +64,18 @@ To use Ray's actor model:
|
||||
futures = [c.read.remote() for c in counters]
|
||||
print(ray.get(futures))
|
||||
|
||||
Visit the `Walkthrough <walkthrough.html>`_ page a more comprehensive overview of Ray features.
|
||||
|
||||
Ray programs can run on a single machine, and can also seamlessly scale to large clusters. To execute the above Ray script in the cloud, just download `this configuration file <https://github.com/ray-project/ray/blob/master/python/ray/autoscaler/aws/example-full.yaml>`__, and run:
|
||||
|
||||
``ray submit [CLUSTER.YAML] example.py --start``
|
||||
|
||||
See more details in the `Cluster Launch page <autoscaling.html>`_.
|
||||
Read more about `launching clusters <autoscaling.html>`_.
|
||||
|
||||
Tune Quick Start
|
||||
----------------
|
||||
|
||||
`Tune`_ is a scalable framework for hyperparameter search built on top of Ray with a focus on deep learning and deep reinforcement learning.
|
||||
`Tune`_ is a library for hyperparameter tuning at any scale. With Tune, you can launch a multi-node distributed hyperparameter sweep in less than 10 lines of code. Tune supports any deep learning framework, including PyTorch, TensorFlow, and Keras.
|
||||
|
||||
.. note::
|
||||
|
||||
@@ -138,28 +145,48 @@ RLlib Quick Start
|
||||
|
||||
.. _`RLlib`: rllib.html
|
||||
|
||||
Contact
|
||||
-------
|
||||
The following are good places to discuss Ray.
|
||||
|
||||
1. `ray-dev@googlegroups.com`_: For discussions about development or any general
|
||||
questions.
|
||||
2. `StackOverflow`_: For questions about how to use Ray.
|
||||
3. `GitHub Issues`_: For bug reports and feature requests.
|
||||
More Information
|
||||
----------------
|
||||
|
||||
- `Tutorial`_
|
||||
- `Blog`_
|
||||
- `Ray paper`_
|
||||
- `Ray HotOS paper`_
|
||||
- `RLlib paper`_
|
||||
- `Tune paper`_
|
||||
|
||||
.. _`Tutorial`: https://github.com/ray-project/tutorial
|
||||
.. _`Blog`: https://ray-project.github.io/
|
||||
.. _`Ray paper`: https://arxiv.org/abs/1712.05889
|
||||
.. _`Ray HotOS paper`: https://arxiv.org/abs/1703.03924
|
||||
.. _`RLlib paper`: https://arxiv.org/abs/1712.09381
|
||||
.. _`Tune paper`: https://arxiv.org/abs/1807.05118
|
||||
|
||||
Getting Involved
|
||||
----------------
|
||||
|
||||
- `ray-dev@googlegroups.com`_: For discussions about development or any general
|
||||
questions.
|
||||
- `StackOverflow`_: For questions about how to use Ray.
|
||||
- `GitHub Issues`_: For reporting bugs and feature requests.
|
||||
- `Pull Requests`_: For submitting code contributions.
|
||||
|
||||
.. _`ray-dev@googlegroups.com`: https://groups.google.com/forum/#!forum/ray-dev
|
||||
.. _`GitHub Issues`: https://github.com/ray-project/ray/issues
|
||||
.. _`StackOverflow`: https://stackoverflow.com/questions/tagged/ray
|
||||
.. _`Pull Requests`: https://github.com/ray-project/ray/pulls
|
||||
|
||||
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
:maxdepth: -1
|
||||
:caption: Installation
|
||||
|
||||
installation.rst
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
:maxdepth: -1
|
||||
:caption: Using Ray
|
||||
|
||||
walkthrough.rst
|
||||
@@ -167,6 +194,7 @@ The following are good places to discuss Ray.
|
||||
using-ray-with-gpus.rst
|
||||
user-profiling.rst
|
||||
inspect.rst
|
||||
object-store.rst
|
||||
configure.rst
|
||||
memory-management.rst
|
||||
advanced.rst
|
||||
@@ -174,7 +202,7 @@ The following are good places to discuss Ray.
|
||||
package-ref.rst
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
:maxdepth: -1
|
||||
:caption: Cluster Setup
|
||||
|
||||
autoscaling.rst
|
||||
@@ -183,7 +211,7 @@ The following are good places to discuss Ray.
|
||||
deploying-on-slurm.rst
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
:maxdepth: -1
|
||||
:caption: Tune
|
||||
|
||||
tune.rst
|
||||
@@ -198,7 +226,7 @@ The following are good places to discuss Ray.
|
||||
tune-contrib.rst
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
:maxdepth: -1
|
||||
:caption: RLlib
|
||||
|
||||
rllib.rst
|
||||
@@ -214,7 +242,7 @@ The following are good places to discuss Ray.
|
||||
rllib-package-ref.rst
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
:maxdepth: -1
|
||||
:caption: Experimental
|
||||
|
||||
distributed_training.rst
|
||||
@@ -224,7 +252,7 @@ The following are good places to discuss Ray.
|
||||
async_api.rst
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
:maxdepth: -1
|
||||
:caption: Examples
|
||||
|
||||
example-rl-pong.rst
|
||||
@@ -235,12 +263,12 @@ The following are good places to discuss Ray.
|
||||
example-lbfgs.rst
|
||||
example-streaming.rst
|
||||
using-ray-with-tensorflow.rst
|
||||
using-ray-with-pytorch.rst
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
:maxdepth: -1
|
||||
:caption: Development and Internals
|
||||
|
||||
install-source.rst
|
||||
development.rst
|
||||
profiling.rst
|
||||
internals-overview.rst
|
||||
|
||||
+10
-15
@@ -27,7 +27,7 @@ For context, when using Ray, several processes are involved.
|
||||
that it can submit tasks to its raylet and get objects from the object
|
||||
store, but it is different in that the raylet will not assign tasks to
|
||||
the driver to be executed.
|
||||
- A **Redis server** maintains much of the system's state. For example, it keeps
|
||||
- Multiple **Redis servers** maintain much of the system's state. For example, it keeps
|
||||
track of which objects live on which machines and of the task specifications
|
||||
(but not data). It can also be queried directly for debugging purposes.
|
||||
|
||||
@@ -42,9 +42,13 @@ To get information about the current nodes in your cluster, you can use ``ray.no
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
>>> import ray
|
||||
>>> ray.init()
|
||||
>>> ray.nodes()
|
||||
import ray
|
||||
|
||||
ray.init()
|
||||
|
||||
print(ray.nodes())
|
||||
|
||||
"""
|
||||
[{'ClientID': 'a9e430719685f3862ed7ba411259d4138f8afb1e',
|
||||
'IsInsertion': True,
|
||||
'NodeManagerAddress': '192.168.19.108',
|
||||
@@ -54,6 +58,7 @@ To get information about the current nodes in your cluster, you can use ``ray.no
|
||||
'RayletSocketName': '/tmp/ray/session_2019-07-28_17-03-53_955034_24883/sockets/raylet',
|
||||
'Resources': {'CPU': 4.0},
|
||||
'alive': True}]
|
||||
"""
|
||||
|
||||
The above information includes:
|
||||
|
||||
@@ -73,15 +78,5 @@ To get information about the current total resource capacity of your cluster, yo
|
||||
|
||||
To get information about the current available resource capacity of your cluster, you can use ``ray.available_resources()``.
|
||||
|
||||
.. autofunction:: ray.cluster_resources
|
||||
.. autofunction:: ray.available_resources
|
||||
:noindex:
|
||||
|
||||
|
||||
Object Information
|
||||
------------------
|
||||
|
||||
To get information about the current objects that have been placed in the Ray object store across the cluster, you can use ``ray.objects()``.
|
||||
|
||||
.. autofunction:: ray.objects
|
||||
:noindex:
|
||||
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
Installing Ray from Source
|
||||
==========================
|
||||
|
||||
If you want to use the latest version of Ray, you can build it from source.
|
||||
Below, we have instructions for building from source for both Linux and MacOS.
|
||||
|
||||
Dependencies
|
||||
~~~~~~~~~~~~
|
||||
|
||||
To build Ray, first install the following dependencies. We recommend using
|
||||
`Anaconda`_.
|
||||
|
||||
.. _`Anaconda`: https://www.continuum.io/downloads
|
||||
|
||||
For Ubuntu, run the following commands:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential curl unzip psmisc
|
||||
|
||||
# If you are not using Anaconda, you need the following.
|
||||
sudo apt-get install python-dev # For Python 2.
|
||||
sudo apt-get install python3-dev # For Python 3.
|
||||
|
||||
pip install cython==0.29.0
|
||||
|
||||
For MacOS, run the following commands:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
brew update
|
||||
brew install wget
|
||||
|
||||
pip install cython==0.29.0
|
||||
|
||||
|
||||
If you are using Anaconda, you may also need to run the following.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
conda install libgcc
|
||||
|
||||
|
||||
Install Ray
|
||||
~~~~~~~~~~~
|
||||
|
||||
Ray can be built from the repository as follows.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
git clone https://github.com/ray-project/ray.git
|
||||
|
||||
# Install Bazel.
|
||||
ray/ci/travis/install-bazel.sh
|
||||
|
||||
cd ray/python
|
||||
pip install -e . --verbose # Add --user if you see a permission denied error.
|
||||
|
||||
Alternatively, Ray can be built from the repository without cloning using pip.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip install git+https://github.com/ray-project/ray.git#subdirectory=python
|
||||
|
||||
Cleaning the source tree
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
The source tree can be cleaned by running
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
git clean -f -f -x -d
|
||||
|
||||
in the ``ray/`` directory. Warning: this command will delete all untracked files
|
||||
and directories and will reset the repository to its checked out state.
|
||||
For a shallower working directory cleanup, you may want to try:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
rm -rf ./build
|
||||
|
||||
under ``ray/``. Incremental builds should work as follows:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pushd ./build && make && popd
|
||||
|
||||
under ``ray/``.
|
||||
|
||||
Docker Source Images
|
||||
--------------------
|
||||
|
||||
Run the script to create Docker images.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
cd ray
|
||||
./build-docker.sh
|
||||
|
||||
This script creates several Docker images:
|
||||
|
||||
- The ``ray-project/deploy`` image is a self-contained copy of code and binaries
|
||||
suitable for end users.
|
||||
- The ``ray-project/examples`` adds additional libraries for running examples.
|
||||
- The ``ray-project/base-deps`` image builds from Ubuntu Xenial and includes
|
||||
Anaconda and other basic dependencies and can serve as a starting point for
|
||||
developers.
|
||||
|
||||
Review images by listing them:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
docker images
|
||||
|
||||
Output should look something like the following:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
REPOSITORY TAG IMAGE ID CREATED SIZE
|
||||
ray-project/examples latest 7584bde65894 4 days ago 3.257 GB
|
||||
ray-project/deploy latest 970966166c71 4 days ago 2.899 GB
|
||||
ray-project/base-deps latest f45d66963151 4 days ago 2.649 GB
|
||||
ubuntu xenial f49eec89601e 3 weeks ago 129.5 MB
|
||||
|
||||
|
||||
Launch Ray in Docker
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Start out by launching the deployment container.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
docker run --shm-size=<shm-size> -t -i ray-project/deploy
|
||||
|
||||
Replace ``<shm-size>`` with a limit appropriate for your system, for example
|
||||
``512M`` or ``2G``. The ``-t`` and ``-i`` options here are required to support
|
||||
interactive use of the container.
|
||||
|
||||
**Note:** Ray requires a **large** amount of shared memory because each object
|
||||
store keeps all of its objects in shared memory, so the amount of shared memory
|
||||
will limit the size of the object store.
|
||||
|
||||
You should now see a prompt that looks something like:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
root@ebc78f68d100:/ray#
|
||||
|
||||
Test if the installation succeeded
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
To test if the installation was successful, try running some tests. This assumes
|
||||
that you've cloned the git repository.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python -m pytest -v python/ray/tests/test_mini.py
|
||||
|
||||
|
||||
Troubleshooting installing Arrow
|
||||
--------------------------------
|
||||
|
||||
Some candidate possibilities.
|
||||
|
||||
You have a different version of Flatbuffers installed
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Arrow pulls and builds its own copy of Flatbuffers, but if you already have
|
||||
Flatbuffers installed, Arrow may find the wrong version. If a directory like
|
||||
``/usr/local/include/flatbuffers`` shows up in the output, this may be the
|
||||
problem. To solve it, get rid of the old version of flatbuffers.
|
||||
|
||||
There is some problem with Boost
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
If a message like ``Unable to find the requested Boost libraries`` appears when
|
||||
installing Arrow, there may be a problem with Boost. This can happen if you
|
||||
installed Boost using MacPorts. This is sometimes solved by using Brew instead.
|
||||
+151
-1
@@ -13,7 +13,7 @@ You can install the latest stable version of Ray as follows.
|
||||
|
||||
pip install -U ray # also recommended: ray[debug]
|
||||
|
||||
Trying snapshots from master
|
||||
Latest Snapshots (Nightlies)
|
||||
----------------------------
|
||||
|
||||
Here are links to the latest wheels (which are built for each commit on the
|
||||
@@ -42,3 +42,153 @@ master branch). To install these wheels, run the following command:
|
||||
.. _`MacOS Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.8.0.dev3-cp36-cp36m-macosx_10_6_intel.whl
|
||||
.. _`MacOS Python 3.5`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.8.0.dev3-cp35-cp35m-macosx_10_6_intel.whl
|
||||
.. _`MacOS Python 2.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.8.0.dev3-cp27-cp27m-macosx_10_6_intel.whl
|
||||
|
||||
Building Ray from Source
|
||||
------------------------
|
||||
|
||||
Installing from ``pip`` should be sufficient for most Ray users.
|
||||
|
||||
However, should you need to build from source, follow instructions below for both Linux and MacOS.
|
||||
|
||||
Dependencies
|
||||
~~~~~~~~~~~~
|
||||
|
||||
To build Ray, first install the following dependencies. We recommend using
|
||||
`Anaconda`_.
|
||||
|
||||
.. _`Anaconda`: https://www.continuum.io/downloads
|
||||
|
||||
For Ubuntu, run the following commands:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential curl unzip psmisc
|
||||
|
||||
# If you are not using Anaconda, you need the following.
|
||||
sudo apt-get install python-dev # For Python 2.
|
||||
sudo apt-get install python3-dev # For Python 3.
|
||||
|
||||
pip install cython==0.29.0
|
||||
|
||||
For MacOS, run the following commands:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
brew update
|
||||
brew install wget
|
||||
|
||||
pip install cython==0.29.0
|
||||
|
||||
|
||||
If you are using Anaconda, you may also need to run the following.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
conda install libgcc
|
||||
|
||||
|
||||
Install Ray
|
||||
~~~~~~~~~~~
|
||||
|
||||
Ray can be built from the repository as follows.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
git clone https://github.com/ray-project/ray.git
|
||||
|
||||
# Install Bazel.
|
||||
ray/ci/travis/install-bazel.sh
|
||||
|
||||
cd ray/python
|
||||
pip install -e . --verbose # Add --user if you see a permission denied error.
|
||||
|
||||
Docker Source Images
|
||||
--------------------
|
||||
|
||||
Run the script to create Docker images.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
cd ray
|
||||
./build-docker.sh
|
||||
|
||||
This script creates several Docker images:
|
||||
|
||||
- The ``ray-project/deploy`` image is a self-contained copy of code and binaries
|
||||
suitable for end users.
|
||||
- The ``ray-project/examples`` adds additional libraries for running examples.
|
||||
- The ``ray-project/base-deps`` image builds from Ubuntu Xenial and includes
|
||||
Anaconda and other basic dependencies and can serve as a starting point for
|
||||
developers.
|
||||
|
||||
Review images by listing them:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
docker images
|
||||
|
||||
Output should look something like the following:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
REPOSITORY TAG IMAGE ID CREATED SIZE
|
||||
ray-project/examples latest 7584bde65894 4 days ago 3.257 GB
|
||||
ray-project/deploy latest 970966166c71 4 days ago 2.899 GB
|
||||
ray-project/base-deps latest f45d66963151 4 days ago 2.649 GB
|
||||
ubuntu xenial f49eec89601e 3 weeks ago 129.5 MB
|
||||
|
||||
|
||||
Launch Ray in Docker
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Start out by launching the deployment container.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
docker run --shm-size=<shm-size> -t -i ray-project/deploy
|
||||
|
||||
Replace ``<shm-size>`` with a limit appropriate for your system, for example
|
||||
``512M`` or ``2G``. The ``-t`` and ``-i`` options here are required to support
|
||||
interactive use of the container.
|
||||
|
||||
**Note:** Ray requires a **large** amount of shared memory because each object
|
||||
store keeps all of its objects in shared memory, so the amount of shared memory
|
||||
will limit the size of the object store.
|
||||
|
||||
You should now see a prompt that looks something like:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
root@ebc78f68d100:/ray#
|
||||
|
||||
Test if the installation succeeded
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
To test if the installation was successful, try running some tests. This assumes
|
||||
that you've cloned the git repository.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python -m pytest -v python/ray/tests/test_mini.py
|
||||
|
||||
|
||||
Troubleshooting installing Arrow
|
||||
--------------------------------
|
||||
|
||||
Some candidate possibilities.
|
||||
|
||||
You have a different version of Flatbuffers installed
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Arrow pulls and builds its own copy of Flatbuffers, but if you already have
|
||||
Flatbuffers installed, Arrow may find the wrong version. If a directory like
|
||||
``/usr/local/include/flatbuffers`` shows up in the output, this may be the
|
||||
problem. To solve it, get rid of the old version of flatbuffers.
|
||||
|
||||
There is some problem with Boost
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
If a message like ``Unable to find the requested Boost libraries`` appears when
|
||||
installing Arrow, there may be a problem with Boost. This can happen if you
|
||||
installed Boost using MacPorts. This is sometimes solved by using Brew instead.
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
Object Store and Serialization
|
||||
==============================
|
||||
|
||||
Since Ray processes do not share memory space, data transferred between workers and nodes will need to **serialized** and **deserialized**. Ray uses the `Plasma object store <https://arrow.apache.org/docs/python/plasma.html>`_ to efficiently transfer objects across different processes and different nodes.
|
||||
|
||||
|
||||
Object Store (Plasma)
|
||||
---------------------
|
||||
|
||||
Plasma is an in-memory object store that is being developed as part of `Apache Arrow`_. Ray uses Plasma to efficiently transfer objects across different processes and different nodes. All objects in Plasma object store are **immutable** and held in shared memory. This is so that they can be accessed efficiently by many workers on the same node.
|
||||
|
||||
Ray will place objects in the object store in the following situations:
|
||||
|
||||
1. Calling ``ray.put``
|
||||
|
||||
.. code:: python
|
||||
|
||||
y = 2
|
||||
# This places `2` into the object store.
|
||||
object_id = ray.put(y)
|
||||
|
||||
2. The return values of a remote function.
|
||||
|
||||
.. code:: python
|
||||
|
||||
@ray.remote
|
||||
def remote_function():
|
||||
return 1
|
||||
|
||||
# This places `1` into the object store.
|
||||
object_id = remote_function.remote()
|
||||
|
||||
|
||||
3. Arguments to remote functions (except for simple arguments like ints or floats).
|
||||
|
||||
.. code:: python
|
||||
|
||||
@ray.remote
|
||||
def remote_function(y):
|
||||
# Note that inside the remote function, the actual argument is provided.
|
||||
return len(y)
|
||||
|
||||
argument = np.random.rand(100, 100)
|
||||
# This implicitly places `argument` into the object store.
|
||||
remote_function.remote(argument)
|
||||
|
||||
Each node has its own object store. When data is put into the object store, it does not get automatically broadcasted to other nodes. Data remains local to the writer until requested by another task or actor on another node.
|
||||
|
||||
|
||||
.. tip:: In certain cases, it may be necessary to either write `your own serialization protocol <object-store.html#custom-serialization>`_ or use Actors to hold objects and transfer object state (i.e., weight matrices) among Ray workers.
|
||||
|
||||
Advanced: Huge Pages
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
On Linux, it is possible to increase the write throughput of the Plasma object store by using huge pages. See the `Configuration page <configure.html#using-the-object-store-with-huge-pages>`_ for information on how to use huge pages in Ray.
|
||||
|
||||
|
||||
.. _`Apache Arrow`: https://arrow.apache.org/
|
||||
|
||||
|
||||
Serialization Overview
|
||||
----------------------
|
||||
|
||||
Objects that are serialized for transfer among Ray processes go through three stages:
|
||||
|
||||
**1. Serialize using pyarrow**: Below is the set of Python objects that Ray can serialize using ``pyarrow``:
|
||||
|
||||
1. Primitive types: ints, floats, longs, bools, strings, unicode, and numpy arrays.
|
||||
|
||||
2. Any list, dictionary, or tuple whose elements can be serialized by Ray.
|
||||
|
||||
**2. ``__dict__`` serialization**: If a direct usage of PyArrow is not possible, Ray will recursively extract the object’s ``__dict__`` and serialize that using pyarrow. This behavior is not correct in all cases.
|
||||
|
||||
**3. Cloudpickle**: Ray falls back to ``cloudpickle`` as a final attempt for serialization. This may be slow.
|
||||
|
||||
Custom Serialization
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
If none of these options work, we recommend registering a custom serializer.
|
||||
|
||||
.. autofunction:: ray.register_custom_serializer
|
||||
:noindex:
|
||||
|
||||
Below is an example of using ``ray.register_custom_serializer``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import ray
|
||||
|
||||
ray.init()
|
||||
|
||||
class Foo(object):
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
|
||||
def custom_serializer(obj):
|
||||
return obj.value
|
||||
|
||||
def custom_deserializer(value):
|
||||
object = Foo()
|
||||
object.value = value
|
||||
return object
|
||||
|
||||
ray.register_custom_serializer(
|
||||
Foo, serializer=custom_serializer, deserializer=custom_deserializer)
|
||||
|
||||
object_id = ray.put(Foo(100))
|
||||
assert ray.get(object_id).value == 100
|
||||
|
||||
|
||||
If you find cases where Ray serialization doesn't work or does something unexpected, please `let us know`_ so we can fix it.
|
||||
|
||||
.. _`let us know`: https://github.com/ray-project/ray/issues
|
||||
|
||||
|
||||
Serialization: Numpy Arrays
|
||||
---------------------------
|
||||
|
||||
Ray optimizes for numpy arrays by using the `Apache Arrow`_ data format.
|
||||
The numpy array is stored as a read-only object, and all Ray workers on the same node can read the numpy array in the object store without copying (zero-copy reads). Each numpy array object in the worker process holds a pointer to the relevant array held in shared memory. Any writes to the read-only object will result in a copy into the local process memory.
|
||||
|
||||
There are some advantages to this form of serialization:
|
||||
|
||||
- Deserialization can be very fast.
|
||||
- Memory is shared between processes so worker processes can all read the same
|
||||
data without having to copy it.
|
||||
|
||||
Serialization notes and limitations
|
||||
-----------------------------------
|
||||
|
||||
- Ray currently handles certain patterns incorrectly, according to Python
|
||||
semantics. For example, a list that contains two copies of the same list will
|
||||
be serialized as if the two lists were distinct.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
l1 = [0]
|
||||
l2 = [l1, l1]
|
||||
l3 = ray.get(ray.put(l2))
|
||||
|
||||
assert l2[0] is l2[1]
|
||||
assert not l3[0] is l3[1]
|
||||
|
||||
- For reasons similar to the above example, we also do not currently handle
|
||||
objects that recursively contain themselves (this may be common in graph-like
|
||||
data structures).
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
l = []
|
||||
l.append(l)
|
||||
|
||||
# Try to put this list that recursively contains itself in the object store.
|
||||
ray.put(l)
|
||||
|
||||
This will throw an exception with a message like the following.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
This object exceeds the maximum recursion depth. It may contain itself recursively.
|
||||
|
||||
- Whenever possible, use numpy arrays for maximum performance.
|
||||
@@ -140,12 +140,9 @@ You may want to get a summary of multiple experiments that point to the same ``l
|
||||
|
||||
See the `full documentation <tune-package-ref.html#ray.tune.Analysis>`_ for the ``Analysis`` object.
|
||||
|
||||
Training Features
|
||||
-----------------
|
||||
|
||||
Tune Search Space (Default)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
---------------------------
|
||||
|
||||
You can use ``tune.grid_search`` to specify an axis of a grid search. By default, Tune also supports sampling parameters from user-specified lambda functions, which can be used independently or in combination with grid search.
|
||||
|
||||
@@ -177,7 +174,7 @@ The following shows grid search over two nested parameters combined with random
|
||||
For more information on variant generation, see `basic_variant.py <https://github.com/ray-project/ray/blob/master/python/ray/tune/suggest/basic_variant.py>`__.
|
||||
|
||||
Custom Trial Names
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
------------------
|
||||
|
||||
To specify custom trial names, you can pass use the ``trial_name_creator`` argument
|
||||
to `tune.run`. This takes a function with the following signature, and
|
||||
@@ -205,7 +202,7 @@ be sure to wrap it with `tune.function`:
|
||||
An example can be found in `logging_example.py <https://github.com/ray-project/ray/blob/master/python/ray/tune/examples/logging_example.py>`__.
|
||||
|
||||
Sampling Multiple Times
|
||||
~~~~~~~~~~~~~~~~~~~~~~~
|
||||
-----------------------
|
||||
|
||||
By default, each random variable and grid search point is sampled once. To take multiple random samples, add ``num_samples: N`` to the experiment config. If `grid_search` is provided as an argument, the grid will be repeated `num_samples` of times.
|
||||
|
||||
@@ -230,7 +227,7 @@ E.g. in the above, ``num_samples=10`` repeats the 3x3 grid search 10 times, for
|
||||
|
||||
|
||||
Using GPUs (Resource Allocation)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
--------------------------------
|
||||
|
||||
Tune will allocate the specified GPU and CPU ``resources_per_trial`` to each individual trial (defaulting to 1 CPU per trial). Under the hood, Tune runs each trial as a Ray actor, using Ray's resource handling to allocate resources and place actors. A trial will not be scheduled unless at least that amount of resources is available in the cluster, preventing the cluster from being overloaded.
|
||||
|
||||
|
||||
+4
-4
@@ -1,17 +1,17 @@
|
||||
Tune: Scalable Hyperparameter Search
|
||||
Tune: Scalable Hyperparameter Tuning
|
||||
====================================
|
||||
|
||||
.. image:: images/tune.png
|
||||
:scale: 30%
|
||||
:align: center
|
||||
|
||||
Tune is a scalable framework for hyperparameter search and model training with a focus on deep learning and deep reinforcement learning.
|
||||
Tune is a library for hyperparameter tuning at any scale.
|
||||
|
||||
* Scale to running on a large distributed cluster without changing your code.
|
||||
* Launch a multi-node Tune experiment in less than 10 lines of code.
|
||||
* Launch a multi-node distributed hyperparameter sweep in less than 10 lines of code.
|
||||
* Supports any deep learning framework, including PyTorch, TensorFlow, and Keras.
|
||||
* Visualize results with `TensorBoard <https://www.tensorflow.org/get_started/summaries_and_tensorboard>`__.
|
||||
* Choose among scalable SOTA algorithms such as `Population Based Training (PBT)`_, `Vizier's Median Stopping Rule`_, `HyperBand/ASHA`_.
|
||||
* Tune integrates with many optimization libraries such as `Facebook Ax <http://ax.dev>`_, `HyperOpt <https://github.com/hyperopt/hyperopt>`_, and `Bayesian Optimization <https://github.com/fmfn/BayesianOptimization>`_ and enables you to scale them transparently.
|
||||
|
||||
.. _`Population Based Training (PBT)`: tune-schedulers.html#population-based-training-pbt
|
||||
.. _`Vizier's Median Stopping Rule`: tune-schedulers.html#median-stopping-rule
|
||||
|
||||
@@ -21,344 +21,11 @@ Then open `chrome://tracing`_ in the Chrome web browser, and load
|
||||
|
||||
.. _`chrome://tracing`: chrome://tracing
|
||||
|
||||
Observing Ray Work
|
||||
------------------
|
||||
|
||||
A Basic Example to Profile
|
||||
--------------------------
|
||||
|
||||
Let's try to profile a simple example, and compare how different ways to
|
||||
write a simple loop can affect performance.
|
||||
|
||||
As a proxy for a computationally intensive and possibly slower function,
|
||||
let's define our remote function to just sleep for 0.5 seconds:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import ray
|
||||
import time
|
||||
|
||||
# Our time-consuming remote function
|
||||
@ray.remote
|
||||
def func():
|
||||
time.sleep(0.5)
|
||||
|
||||
In our example setup, we wish to call our remote function ``func()`` five
|
||||
times, and store the result of each call into a list. To compare the
|
||||
performance of different ways of looping our calls to our remote function,
|
||||
we can define each loop version as a separate function on the driver script.
|
||||
|
||||
For the first version **ex1**, each iteration of the loop calls the remote
|
||||
function, then calls ``ray.get`` in an attempt to store the current result
|
||||
into the list, as follows:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# This loop is suboptimal in Ray, and should only be used for the sake of this example
|
||||
def ex1():
|
||||
list1 = []
|
||||
for i in range(5):
|
||||
list1.append(ray.get(func.remote()))
|
||||
|
||||
For the second version **ex2**, each iteration of the loop calls the remote
|
||||
function, and stores it into the list **without** calling ``ray.get`` each time.
|
||||
``ray.get`` is used after the loop has finished, in preparation for processing
|
||||
``func()``'s results:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# This loop is more proper in Ray
|
||||
def ex2():
|
||||
list2 = []
|
||||
for i in range(5):
|
||||
list2.append(func.remote())
|
||||
ray.get(list2)
|
||||
|
||||
Finally, for an example that's not so parallelizable, let's create a
|
||||
third version **ex3** where the driver has to call a local
|
||||
function in between each call to the remote function ``func()``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# A local function executed on the driver, not on Ray
|
||||
def other_func():
|
||||
time.sleep(0.3)
|
||||
|
||||
def ex3():
|
||||
list3 = []
|
||||
for i in range(5):
|
||||
other_func()
|
||||
list3.append(func.remote())
|
||||
ray.get(list3)
|
||||
|
||||
|
||||
Timing Performance Using Python's Timestamps
|
||||
--------------------------------------------
|
||||
|
||||
One way to sanity-check the performance of the three loops is simply to
|
||||
time how long it takes to complete each loop version. We can do this using
|
||||
python's built-in ``time`` `module`_.
|
||||
|
||||
.. _`module`: https://docs.python.org/3/library/time.html
|
||||
|
||||
The ``time`` module contains a useful ``time()`` function that returns the
|
||||
current timestamp in unix time whenever it's called. We can create a generic
|
||||
function wrapper to call ``time()`` right before and right after each loop
|
||||
function to print out how long each loop takes overall:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# This is a generic wrapper for any driver function you want to time
|
||||
def time_this(f):
|
||||
def timed_wrapper(*args, **kw):
|
||||
start_time = time.time()
|
||||
result = f(*args, **kw)
|
||||
end_time = time.time()
|
||||
|
||||
# Time taken = end_time - start_time
|
||||
print('| func:%r args:[%r, %r] took: %2.4f seconds |' % \
|
||||
(f.__name__, args, kw, end_time - start_time))
|
||||
return result
|
||||
return timed_wrapper
|
||||
|
||||
To always print out how long the loop takes to run each time the loop
|
||||
function ``ex1()`` is called, we can evoke our ``time_this`` wrapper with
|
||||
a function decorator. This can similarly be done to functions ``ex2()``
|
||||
and ``ex3()``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@time_this # Added decorator
|
||||
def ex1():
|
||||
list1 = []
|
||||
for i in range(5):
|
||||
list1.append(ray.get(func.remote()))
|
||||
|
||||
def main():
|
||||
ray.init()
|
||||
ex1()
|
||||
ex2()
|
||||
ex3()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Then, running the three timed loops should yield output similar to this:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
| func:'ex1' args:[(), {}] took: 2.5083 seconds |
|
||||
| func:'ex2' args:[(), {}] took: 1.0032 seconds |
|
||||
| func:'ex3' args:[(), {}] took: 2.0039 seconds |
|
||||
|
||||
Let's interpret these results.
|
||||
|
||||
Here, ``ex1()`` took substantially more time than ``ex2()``, where
|
||||
their only difference is that ``ex1()`` calls ``ray.get`` on the remote
|
||||
function before adding it to the list, while ``ex2()`` waits to fetch the
|
||||
entire list with ``ray.get`` at once.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@ray.remote
|
||||
def func(): # A single call takes 0.5 seconds
|
||||
time.sleep(0.5)
|
||||
|
||||
def ex1(): # Took Ray 2.5 seconds
|
||||
list1 = []
|
||||
for i in range(5):
|
||||
list1.append(ray.get(func.remote()))
|
||||
|
||||
def ex2(): # Took Ray 1 second
|
||||
list2 = []
|
||||
for i in range(5):
|
||||
list2.append(func.remote())
|
||||
ray.get(list2)
|
||||
|
||||
Notice how ``ex1()`` took 2.5 seconds, exactly five times 0.5 seconds, or
|
||||
the time it would take to wait for our remote function five times in a row.
|
||||
|
||||
By calling ``ray.get`` after each call to the remote function, ``ex1()``
|
||||
removes all ability to parallelize work, by forcing the driver to wait for
|
||||
each ``func()``'s result in succession. We are not taking advantage of Ray
|
||||
parallelization here!
|
||||
|
||||
Meanwhile, ``ex2()`` takes about 1 second, much faster than it would normally
|
||||
take to call ``func()`` five times iteratively. Ray is running each call to
|
||||
``func()`` in parallel, saving us time.
|
||||
|
||||
``ex1()`` is actually a common user mistake in Ray. ``ray.get`` is not
|
||||
necessary to do before adding the result of ``func()`` to the list. Instead,
|
||||
the driver should send out all parallelizable calls to the remote function
|
||||
to Ray before waiting to receive their results with ``ray.get``. ``ex1()``'s
|
||||
suboptimal behavior can be noticed just using this simple timing test.
|
||||
|
||||
Realistically, however, many applications are not as highly parallelizable
|
||||
as ``ex2()``, and the application includes sections where the code must run in
|
||||
serial. ``ex3()`` is such an example, where the local function ``other_func()``
|
||||
must run first before each call to ``func()`` can be submitted to Ray.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# A local function that must run in serial
|
||||
def other_func():
|
||||
time.sleep(0.3)
|
||||
|
||||
def ex3(): # Took Ray 2 seconds, vs. ex1 taking 2.5 seconds
|
||||
list3 = []
|
||||
for i in range(5):
|
||||
other_func()
|
||||
list2.append(func.remote())
|
||||
ray.get(list3)
|
||||
|
||||
What results is that while ``ex3()`` still gained 0.5 seconds of speedup
|
||||
compared to the completely serialized ``ex1()`` version, this speedup is
|
||||
still nowhere near the ideal speedup of ``ex2()``.
|
||||
|
||||
The dramatic speedup of ``ex2()`` is possible because ``ex2()`` is
|
||||
theoretically completely parallelizable: if we were given 5 CPUs, all 5 calls
|
||||
to ``func()`` can be run in parallel. What is happening with ``ex3()``,
|
||||
however, is that each parallelized call to ``func()`` is staggered by a wait
|
||||
of 0.3 seconds for the local ``other_func()`` to finish.
|
||||
|
||||
``ex3()`` is thus a manifestation of `Amdahls Law`_: the fastest theoretically
|
||||
possible execution time from parallelizing an application is limited to be
|
||||
no better than the time it takes to run all serial parts in serial.
|
||||
|
||||
.. _`Amdahls Law`: https://en.wikipedia.org/wiki/Amdahl%27s_law
|
||||
|
||||
Due to Amdahl's Law, ``ex3()`` must take at least 1.5
|
||||
seconds -- the time it takes for 5 serial calls to ``other_func()`` to finish!
|
||||
After an additional 0.5 seconds to execute func and get the result, the
|
||||
computation is done.
|
||||
|
||||
|
||||
Profiling Using An External Profiler (Line Profiler)
|
||||
----------------------------------------------------
|
||||
|
||||
One way to profile the performance of our code using Ray is to use a third-party
|
||||
profiler such as `Line_profiler`_. Line_profiler is a useful line-by-line
|
||||
profiler for pure Python applications that formats its output side-by-side with
|
||||
the profiled code itself.
|
||||
|
||||
Alternatively, another third-party profiler (not covered in this documentation)
|
||||
that you could use is `Pyflame`_, which can generate profiling graphs.
|
||||
|
||||
.. _`Line_profiler`: https://github.com/rkern/line_profiler
|
||||
.. _`Pyflame`: https://github.com/uber/pyflame
|
||||
|
||||
First install ``line_profiler`` with pip:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip install line_profiler
|
||||
|
||||
``line_profiler`` requires each section of driver code that you want to profile as
|
||||
its own independent function. Conveniently, we have already done so by defining
|
||||
each loop version as its own function. To tell ``line_profiler`` which functions
|
||||
to profile, just add the ``@profile`` decorator to ``ex1()``, ``ex2()`` and
|
||||
``ex3()``. Note that you do not need to import ``line_profiler`` into your Ray
|
||||
application:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@profile # Added decorator
|
||||
def ex1():
|
||||
list1 = []
|
||||
for i in range(5):
|
||||
list1.append(ray.get(func.remote()))
|
||||
|
||||
def main():
|
||||
ray.init()
|
||||
ex1()
|
||||
ex2()
|
||||
ex3()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Then, when we want to execute our Python script from the command line, instead
|
||||
of ``python your_script_here.py``, we use the following shell command to run the
|
||||
script with ``line_profiler`` enabled:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
kernprof -l your_script_here.py
|
||||
|
||||
This command runs your script and prints only your script's output as usual.
|
||||
``Line_profiler`` instead outputs its profiling results to a corresponding
|
||||
binary file called ``your_script_here.py.lprof``.
|
||||
|
||||
To read ``line_profiler``'s results to terminal, use this shell command:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python -m line_profiler your_script_here.py.lprof
|
||||
|
||||
In our loop example, this command outputs results for ``ex1()`` as follows.
|
||||
Note that execution time is given in units of 1e-06 seconds:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
Timer unit: 1e-06 s
|
||||
|
||||
Total time: 2.50883 s
|
||||
File: your_script_here.py
|
||||
Function: ex1 at line 28
|
||||
|
||||
Line # Hits Time Per Hit % Time Line Contents
|
||||
==============================================================
|
||||
29 @profile
|
||||
30 def ex1():
|
||||
31 1 3.0 3.0 0.0 list1 = []
|
||||
32 6 18.0 3.0 0.0 for i in range(5):
|
||||
33 5 2508805.0 501761.0 100.0 list1.append(ray.get(func.remote()))
|
||||
|
||||
|
||||
Notice that each hit to ``list1.append(ray.get(func.remote()))`` at line 33
|
||||
takes the full 0.5 seconds waiting for ``func()`` to finish. Meanwhile, in
|
||||
``ex2()`` below, each call of ``func.remote()`` at line 40 only takes 0.127 ms,
|
||||
and the majority of the time (about 1 second) is spent on waiting for ``ray.get()``
|
||||
at the end:
|
||||
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
Total time: 1.00357 s
|
||||
File: your_script_here.py
|
||||
Function: ex2 at line 35
|
||||
|
||||
Line # Hits Time Per Hit % Time Line Contents
|
||||
==============================================================
|
||||
36 @profile
|
||||
37 def ex2():
|
||||
38 1 2.0 2.0 0.0 list2 = []
|
||||
39 6 13.0 2.2 0.0 for i in range(5):
|
||||
40 5 637.0 127.4 0.1 list2.append(func.remote())
|
||||
41 1 1002919.0 1002919.0 99.9 ray.get(list2)
|
||||
|
||||
|
||||
And finally, ``line_profiler``'s output for ``ex3()``. Each call to
|
||||
``func.remote()`` at line 50 still take magnitudes faster than 0.5 seconds,
|
||||
showing that Ray is successfully parallelizing the remote calls. However, each
|
||||
call to the local function ``other_func()`` takes the full 0.3 seconds,
|
||||
totalling up to the guaranteed minimum application execution time of 1.5
|
||||
seconds:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
Total time: 2.00446 s
|
||||
File: basic_kernprof.py
|
||||
Function: ex3 at line 44
|
||||
|
||||
Line # Hits Time Per Hit % Time Line Contents
|
||||
==============================================================
|
||||
44 @profile
|
||||
45 #@time_this
|
||||
46 def ex3():
|
||||
47 1 2.0 2.0 0.0 list3 = []
|
||||
48 6 13.0 2.2 0.0 for i in range(5):
|
||||
49 5 1501934.0 300386.8 74.9 other_func()
|
||||
50 5 917.0 183.4 0.0 list3.append(func.remote())
|
||||
51 1 501589.0 501589.0 25.0 ray.get(list3)
|
||||
You can run ``ray stack`` to dump the stack traces of all Ray workers on
|
||||
the current node. This requires ``py-spy`` to be installed. See the `Troubleshooting page <troubleshooting.html>`_ for more details.
|
||||
|
||||
|
||||
Profiling Using Python's CProfile
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
Best Practices: Ray with PyTorch
|
||||
================================
|
||||
|
||||
This document describes best practices for using Ray with PyTorch. Feel free to contribute if you think this document is missing anything.
|
||||
|
||||
Downloading Data
|
||||
----------------
|
||||
|
||||
It is very common for multiple Ray actors running PyTorch to have code that downloads the dataset for training and testing.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# This is running inside a Ray actor
|
||||
# ...
|
||||
torch.utils.data.DataLoader(
|
||||
datasets.MNIST(
|
||||
"../data", train=True, download=True,
|
||||
transform=transforms.Compose([
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize((0.1307,), (0.3081,))
|
||||
])),
|
||||
128, shuffle=True, **kwargs)
|
||||
# ...
|
||||
|
||||
This may cause different processes to simultaneously download the data and cause data corruption. One easy workaround for this is to use ``Filelock``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from filelock import FileLock
|
||||
|
||||
with FileLock("./data.lock"):
|
||||
torch.utils.data.DataLoader(
|
||||
datasets.MNIST(
|
||||
"./data", train=True, download=True,
|
||||
transform=transforms.Compose([
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize((0.1307,), (0.3081,))
|
||||
])),
|
||||
128, shuffle=True, **kwargs)
|
||||
|
||||
|
||||
Use Actors for Parallel Models
|
||||
------------------------------
|
||||
|
||||
One common use case for using Ray with PyTorch is to parallelize the training of multiple models.
|
||||
|
||||
.. tip::
|
||||
Avoid sending the PyTorch model directly. Send ``model.state_dict()``, as
|
||||
PyTorch tensors are natively supported by the Plasma Object Store.
|
||||
|
||||
|
||||
Suppose we have a simple network definition (this one is modified from the
|
||||
PyTorch documentation).
|
||||
|
||||
.. literalinclude:: ../examples/doc_code/torch_example.py
|
||||
:language: python
|
||||
:start-after: __torch_model_start__
|
||||
:end-before: __torch_model_end__
|
||||
|
||||
Along with these helper training functions:
|
||||
|
||||
.. literalinclude:: ../examples/doc_code/torch_example.py
|
||||
:language: python
|
||||
:start-after: __torch_helper_start__
|
||||
:end-before: __torch_helper_end
|
||||
|
||||
Let's now define a class that captures the training process.
|
||||
|
||||
.. literalinclude:: ../examples/doc_code/torch_example.py
|
||||
:language: python
|
||||
:start-after: __torch_net_start__
|
||||
:end-before: __torch_net_end
|
||||
|
||||
|
||||
To train multiple models, you can convert the above class into a Ray Actor class.
|
||||
|
||||
.. literalinclude:: ../examples/doc_code/torch_example.py
|
||||
:language: python
|
||||
:start-after: __torch_ray_start__
|
||||
:end-before: __torch_ray_end__
|
||||
|
||||
|
||||
Then, we can instantiate multiple copies of the Model, each running on different processes. If GPU is enabled, each copy runs on a different GPU. See the `GPU guide <using-ray-with-gpus.html>`_ for more information.
|
||||
|
||||
.. literalinclude:: ../examples/doc_code/torch_example.py
|
||||
:language: python
|
||||
:start-after: __torch_actor_start__
|
||||
:end-before: __torch_actor_end__
|
||||
|
||||
|
||||
We can then use ``set_weights`` and ``get_weights`` to move the weights of the neural network around. The below example averages the weights of the two networks and sends them back to update the original actors.
|
||||
|
||||
|
||||
.. literalinclude:: ../examples/doc_code/torch_example.py
|
||||
:language: python
|
||||
:start-after: __weight_average_start__
|
||||
@@ -1,31 +1,57 @@
|
||||
Using Ray with TensorFlow
|
||||
=========================
|
||||
Best Practices: Ray with Tensorflow
|
||||
===================================
|
||||
|
||||
This document describes best practices for using Ray with TensorFlow.
|
||||
|
||||
To see more involved examples using TensorFlow, take a look at
|
||||
`A3C`_, `ResNet`_, and `LBFGS`_.
|
||||
|
||||
.. _`A3C`: http://ray.readthedocs.io/en/latest/example-a3c.html
|
||||
.. _`ResNet`: http://ray.readthedocs.io/en/latest/example-resnet.html
|
||||
.. _`LBFGS`: http://ray.readthedocs.io/en/latest/example-lbfgs.html
|
||||
This document describes best practices for using Ray with TensorFlow. Feel free to contribute if you think this document is missing anything.
|
||||
|
||||
Use Actors for Parallel Models
|
||||
------------------------------
|
||||
|
||||
If you are training a deep network in the distributed setting, you may need to
|
||||
ship your deep network between processes (or machines). However, shipping the model is not always straightforward.
|
||||
|
||||
A straightforward attempt to pickle a TensorFlow graph gives mixed
|
||||
results. Some examples fail, and some succeed (but produce very large strings).
|
||||
The results are similar with other pickling libraries as well.
|
||||
.. tip::
|
||||
Avoid sending the Tensorflow model directly. A straightforward attempt to pickle a TensorFlow graph gives mixed results. Furthermore, creating a TensorFlow graph can take tens of seconds, and so serializing a graph and recreating it in another process will be inefficient.
|
||||
|
||||
Furthermore, creating a TensorFlow graph can take tens of seconds, and so
|
||||
serializing a graph and recreating it in another process will be inefficient.
|
||||
The better solution is to replicate the same TensorFlow graph on each worker once
|
||||
It is recommended to replicate the same TensorFlow graph on each worker once
|
||||
at the beginning and then to ship only the weights between the workers.
|
||||
|
||||
Suppose we have a simple network definition (this one is modified from the
|
||||
TensorFlow documentation).
|
||||
|
||||
.. literalinclude:: ../examples/doc_code/tf_example.py
|
||||
:language: python
|
||||
:start-after: __tf_model_start__
|
||||
:end-before: __tf_model_end__
|
||||
|
||||
It is strongly recommended you create actors to handle this. To do this, first initialize
|
||||
ray and define an Actor class:
|
||||
|
||||
.. literalinclude:: ../examples/doc_code/tf_example.py
|
||||
:language: python
|
||||
:start-after: __ray_start__
|
||||
:end-before: __ray_end__
|
||||
|
||||
Then, we can instantiate this actor and train it on the separate process:
|
||||
|
||||
.. literalinclude:: ../examples/doc_code/tf_example.py
|
||||
:language: python
|
||||
:start-after: __actor_start__
|
||||
:end-before: __actor_end__
|
||||
|
||||
|
||||
We can then use ``set_weights`` and ``get_weights`` to move the weights of the neural network
|
||||
around. This allows us to manipulate weights between different models running in parallel without shipping the actual TensorFlow graphs, which are much more complex Python objects.
|
||||
|
||||
|
||||
.. literalinclude:: ../examples/doc_code/tf_example.py
|
||||
:language: python
|
||||
:start-after: __weight_average_start__
|
||||
|
||||
|
||||
Lower-level TF Utilities
|
||||
------------------------
|
||||
Given a low-level TF definition:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import tensorflow as tf
|
||||
@@ -62,6 +88,7 @@ network as follows.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
sess = tf.Session()
|
||||
# First initialize the weights.
|
||||
sess.run(init)
|
||||
# Get the weights
|
||||
@@ -78,251 +105,16 @@ unmanageably large over time.
|
||||
w.assign(np.zeros(1)) # This adds a node to the graph every time you call it.
|
||||
b.assign(np.zeros(1)) # This adds a node to the graph every time you call it.
|
||||
|
||||
Complete Example for Weight Averaging
|
||||
-------------------------------------
|
||||
|
||||
Putting this all together, we would first embed the graph in an actor. Within
|
||||
the actor, we would use the ``get_weights`` and ``set_weights`` methods of the
|
||||
``TensorFlowVariables`` class. We would then use those methods to ship the weights
|
||||
(as a dictionary of variable names mapping to numpy arrays) between the
|
||||
processes without shipping the actual TensorFlow graphs, which are much more
|
||||
complex Python objects.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import tensorflow as tf
|
||||
import numpy as np
|
||||
import ray
|
||||
import ray.experimental.tf_utils
|
||||
|
||||
ray.init()
|
||||
|
||||
BATCH_SIZE = 100
|
||||
NUM_BATCHES = 1
|
||||
NUM_ITERS = 201
|
||||
|
||||
class Network(object):
|
||||
def __init__(self, x, y):
|
||||
# Seed TensorFlow to make the script deterministic.
|
||||
tf.set_random_seed(0)
|
||||
# Define the inputs.
|
||||
self.x_data = tf.constant(x, dtype=tf.float32)
|
||||
self.y_data = tf.constant(y, dtype=tf.float32)
|
||||
# Define the weights and computation.
|
||||
w = tf.Variable(tf.random_uniform([1], -1.0, 1.0))
|
||||
b = tf.Variable(tf.zeros([1]))
|
||||
y = w * self.x_data + b
|
||||
# Define the loss.
|
||||
self.loss = tf.reduce_mean(tf.square(y - self.y_data))
|
||||
optimizer = tf.train.GradientDescentOptimizer(0.5)
|
||||
self.grads = optimizer.compute_gradients(self.loss)
|
||||
self.train = optimizer.apply_gradients(self.grads)
|
||||
# Define the weight initializer and session.
|
||||
init = tf.global_variables_initializer()
|
||||
self.sess = tf.Session()
|
||||
# Additional code for setting and getting the weights
|
||||
self.variables = ray.experimental.tf_utils.TensorFlowVariables(self.loss, self.sess)
|
||||
# Return all of the data needed to use the network.
|
||||
self.sess.run(init)
|
||||
|
||||
# Define a remote function that trains the network for one step and returns the
|
||||
# new weights.
|
||||
def step(self, weights):
|
||||
# Set the weights in the network.
|
||||
self.variables.set_weights(weights)
|
||||
# Do one step of training.
|
||||
self.sess.run(self.train)
|
||||
# Return the new weights.
|
||||
return self.variables.get_weights()
|
||||
|
||||
def get_weights(self):
|
||||
return self.variables.get_weights()
|
||||
|
||||
# Define a remote function for generating fake data.
|
||||
@ray.remote(num_return_vals=2)
|
||||
def generate_fake_x_y_data(num_data, seed=0):
|
||||
# Seed numpy to make the script deterministic.
|
||||
np.random.seed(seed)
|
||||
x = np.random.rand(num_data)
|
||||
y = x * 0.1 + 0.3
|
||||
return x, y
|
||||
|
||||
# Generate some training data.
|
||||
batch_ids = [generate_fake_x_y_data.remote(BATCH_SIZE, seed=i) for i in range(NUM_BATCHES)]
|
||||
x_ids = [x_id for x_id, y_id in batch_ids]
|
||||
y_ids = [y_id for x_id, y_id in batch_ids]
|
||||
# Generate some test data.
|
||||
x_test, y_test = ray.get(generate_fake_x_y_data.remote(BATCH_SIZE, seed=NUM_BATCHES))
|
||||
|
||||
# Create actors to store the networks.
|
||||
remote_network = ray.remote(Network)
|
||||
actor_list = [remote_network.remote(x_ids[i], y_ids[i]) for i in range(NUM_BATCHES)]
|
||||
|
||||
# Get initial weights of some actor.
|
||||
weights = ray.get(actor_list[0].get_weights.remote())
|
||||
|
||||
# Do some steps of training.
|
||||
for iteration in range(NUM_ITERS):
|
||||
# Put the weights in the object store. This is optional. We could instead pass
|
||||
# the variable weights directly into step.remote, in which case it would be
|
||||
# placed in the object store under the hood. However, in that case multiple
|
||||
# copies of the weights would be put in the object store, so this approach is
|
||||
# more efficient.
|
||||
weights_id = ray.put(weights)
|
||||
# Call the remote function multiple times in parallel.
|
||||
new_weights_ids = [actor.step.remote(weights_id) for actor in actor_list]
|
||||
# Get all of the weights.
|
||||
new_weights_list = ray.get(new_weights_ids)
|
||||
# Add up all the different weights. Each element of new_weights_list is a dict
|
||||
# of weights, and we want to add up these dicts component wise using the keys
|
||||
# of the first dict.
|
||||
weights = {variable: sum(weight_dict[variable] for weight_dict in new_weights_list) / NUM_BATCHES for variable in new_weights_list[0]}
|
||||
# Print the current weights. They should converge to roughly to the values 0.1
|
||||
# and 0.3 used in generate_fake_x_y_data.
|
||||
if iteration % 20 == 0:
|
||||
print("Iteration {}: weights are {}".format(iteration, weights))
|
||||
|
||||
How to Train in Parallel using Ray and Gradients
|
||||
------------------------------------------------
|
||||
|
||||
In some cases, you may want to do data-parallel training on your network. We use the network
|
||||
above to illustrate how to do this in Ray. The only differences are in the remote function
|
||||
``step`` and the driver code.
|
||||
|
||||
In the function ``step``, we run the grad operation rather than the train operation to get the gradients.
|
||||
Since Tensorflow pairs the gradients with the variables in a tuple, we extract the gradients to avoid
|
||||
needless computation.
|
||||
|
||||
Extracting numerical gradients
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Code like the following can be used in a remote function to compute numerical gradients.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
x_values = [1] * 100
|
||||
y_values = [2] * 100
|
||||
numerical_grads = sess.run([grad[0] for grad in grads], feed_dict={x_data: x_values, y_data: y_values})
|
||||
|
||||
Using the returned gradients to train the network
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
By pairing the symbolic gradients with the numerical gradients in a feed_dict, we can update the network.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# We can feed the gradient values in using the associated symbolic gradient
|
||||
# operation defined in tensorflow.
|
||||
feed_dict = {grad[0]: numerical_grad for (grad, numerical_grad) in zip(grads, numerical_grads)}
|
||||
sess.run(train, feed_dict=feed_dict)
|
||||
|
||||
You can then run ``variables.get_weights()`` to see the updated weights of the network.
|
||||
|
||||
For reference, the full code is below:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import tensorflow as tf
|
||||
import numpy as np
|
||||
import ray
|
||||
|
||||
ray.init()
|
||||
|
||||
BATCH_SIZE = 100
|
||||
NUM_BATCHES = 1
|
||||
NUM_ITERS = 201
|
||||
|
||||
class Network(object):
|
||||
def __init__(self, x, y):
|
||||
# Seed TensorFlow to make the script deterministic.
|
||||
tf.set_random_seed(0)
|
||||
# Define the inputs.
|
||||
x_data = tf.constant(x, dtype=tf.float32)
|
||||
y_data = tf.constant(y, dtype=tf.float32)
|
||||
# Define the weights and computation.
|
||||
w = tf.Variable(tf.random_uniform([1], -1.0, 1.0))
|
||||
b = tf.Variable(tf.zeros([1]))
|
||||
y = w * x_data + b
|
||||
# Define the loss.
|
||||
self.loss = tf.reduce_mean(tf.square(y - y_data))
|
||||
optimizer = tf.train.GradientDescentOptimizer(0.5)
|
||||
self.grads = optimizer.compute_gradients(self.loss)
|
||||
self.train = optimizer.apply_gradients(self.grads)
|
||||
# Define the weight initializer and session.
|
||||
init = tf.global_variables_initializer()
|
||||
self.sess = tf.Session()
|
||||
# Additional code for setting and getting the weights
|
||||
self.variables = ray.experimental.tf_utils.TensorFlowVariables(self.loss, self.sess)
|
||||
# Return all of the data needed to use the network.
|
||||
self.sess.run(init)
|
||||
|
||||
# Define a remote function that trains the network for one step and returns the
|
||||
# new weights.
|
||||
def step(self, weights):
|
||||
# Set the weights in the network.
|
||||
self.variables.set_weights(weights)
|
||||
# Do one step of training. We only need the actual gradients so we filter over the list.
|
||||
actual_grads = self.sess.run([grad[0] for grad in self.grads])
|
||||
return actual_grads
|
||||
|
||||
def get_weights(self):
|
||||
return self.variables.get_weights()
|
||||
|
||||
# Define a remote function for generating fake data.
|
||||
@ray.remote(num_return_vals=2)
|
||||
def generate_fake_x_y_data(num_data, seed=0):
|
||||
# Seed numpy to make the script deterministic.
|
||||
np.random.seed(seed)
|
||||
x = np.random.rand(num_data)
|
||||
y = x * 0.1 + 0.3
|
||||
return x, y
|
||||
|
||||
# Generate some training data.
|
||||
batch_ids = [generate_fake_x_y_data.remote(BATCH_SIZE, seed=i) for i in range(NUM_BATCHES)]
|
||||
x_ids = [x_id for x_id, y_id in batch_ids]
|
||||
y_ids = [y_id for x_id, y_id in batch_ids]
|
||||
# Generate some test data.
|
||||
x_test, y_test = ray.get(generate_fake_x_y_data.remote(BATCH_SIZE, seed=NUM_BATCHES))
|
||||
|
||||
# Create actors to store the networks.
|
||||
remote_network = ray.remote(Network)
|
||||
actor_list = [remote_network.remote(x_ids[i], y_ids[i]) for i in range(NUM_BATCHES)]
|
||||
local_network = Network(x_test, y_test)
|
||||
|
||||
# Get initial weights of local network.
|
||||
weights = local_network.get_weights()
|
||||
|
||||
# Do some steps of training.
|
||||
for iteration in range(NUM_ITERS):
|
||||
# Put the weights in the object store. This is optional. We could instead pass
|
||||
# the variable weights directly into step.remote, in which case it would be
|
||||
# placed in the object store under the hood. However, in that case multiple
|
||||
# copies of the weights would be put in the object store, so this approach is
|
||||
# more efficient.
|
||||
weights_id = ray.put(weights)
|
||||
# Call the remote function multiple times in parallel.
|
||||
gradients_ids = [actor.step.remote(weights_id) for actor in actor_list]
|
||||
# Get all of the weights.
|
||||
gradients_list = ray.get(gradients_ids)
|
||||
|
||||
# Take the mean of the different gradients. Each element of gradients_list is a list
|
||||
# of gradients, and we want to take the mean of each one.
|
||||
mean_grads = [sum([gradients[i] for gradients in gradients_list]) / len(gradients_list) for i in range(len(gradients_list[0]))]
|
||||
|
||||
feed_dict = {grad[0]: mean_grad for (grad, mean_grad) in zip(local_network.grads, mean_grads)}
|
||||
local_network.sess.run(local_network.train, feed_dict=feed_dict)
|
||||
weights = local_network.get_weights()
|
||||
|
||||
# Print the current weights. They should converge to roughly to the values 0.1
|
||||
# and 0.3 used in generate_fake_x_y_data.
|
||||
if iteration % 20 == 0:
|
||||
print("Iteration {}: weights are {}".format(iteration, weights))
|
||||
|
||||
.. autoclass:: ray.experimental.tf_utils.TensorFlowVariables
|
||||
:members:
|
||||
|
||||
.. note:: This may not work with `tf.Keras`.
|
||||
|
||||
|
||||
|
||||
Troubleshooting
|
||||
---------------
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
Note that ``TensorFlowVariables`` uses variable names to determine what
|
||||
variables to set when calling ``set_weights``. One common issue arises when two
|
||||
|
||||
+19
-29
@@ -47,14 +47,12 @@ This causes a few things changes in behavior:
|
||||
|
||||
.. code:: python
|
||||
|
||||
>>> regular_function()
|
||||
1
|
||||
assert regular_function() == 1
|
||||
|
||||
>>> remote_function.remote()
|
||||
ObjectID(1c80d6937802cd7786ad25e50caf2f023c95e350)
|
||||
object_id = remote_function.remote()
|
||||
|
||||
>>> ray.get(remote_function.remote())
|
||||
1
|
||||
# The value of the original `regular_function`
|
||||
assert ray.get(object_id) == 1
|
||||
|
||||
3. **Parallelism:** Invocations of ``regular_function`` happen
|
||||
**serially**, for example
|
||||
@@ -76,17 +74,20 @@ This causes a few things changes in behavior:
|
||||
|
||||
See the `ray.remote package reference <package-ref.html>`__ page for specific documentation on how to use ``ray.remote``.
|
||||
|
||||
**Object IDs** can also be passed into remote functions. When the function actually gets executed, **the argument will be a retrieved as a regular Python object**.
|
||||
**Object IDs** can also be passed into remote functions. When the function actually gets executed, **the argument will be a retrieved as a regular Python object**. For example, take this function:
|
||||
|
||||
.. code:: python
|
||||
|
||||
>>> y1_id = f.remote(x1_id)
|
||||
>>> ray.get(y1_id)
|
||||
1
|
||||
@ray.remote
|
||||
def remote_chain_function(value):
|
||||
return value + 1
|
||||
|
||||
>>> y2_id = f.remote(x2_id)
|
||||
>>> ray.get(y2_id)
|
||||
[1, 2, 3]
|
||||
|
||||
y1_id = remote_function.remote()
|
||||
assert ray.get(y1_id) == 1
|
||||
|
||||
chained_id = remote_chain_function.remote(y1_id)
|
||||
assert ray.get(chained_id) == 2
|
||||
|
||||
|
||||
Note the following behaviors:
|
||||
@@ -171,12 +172,8 @@ Object IDs can be created in multiple ways.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
>>> y = 1
|
||||
>>> y_id = ray.put(y)
|
||||
>>> print(y_id)
|
||||
ObjectID(0369a14bc595e08cfbd508dfaa162cb7feffffff)
|
||||
|
||||
Here is the docstring for ``ray.put``:
|
||||
y = 1
|
||||
object_id = ray.put(y)
|
||||
|
||||
.. autofunction:: ray.put
|
||||
:noindex:
|
||||
@@ -198,14 +195,9 @@ shared memory and avoid copying the object.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
>>> y = 1
|
||||
>>> obj_id = ray.put(y)
|
||||
>>> print(obj_id)
|
||||
ObjectID(0369a14bc595e08cfbd508dfaa162cb7feffffff)
|
||||
>>> ray.get(obj_id)
|
||||
1
|
||||
|
||||
Here is the docstring for ``ray.get``:
|
||||
y = 1
|
||||
obj_id = ray.put(y)
|
||||
assert ray.get(obj_id) == 1
|
||||
|
||||
.. autofunction:: ray.get
|
||||
:noindex:
|
||||
@@ -219,8 +211,6 @@ works as follows.
|
||||
|
||||
ready_ids, remaining_ids = ray.wait(object_ids, num_returns=1, timeout=None)
|
||||
|
||||
Here is the docstring for ``ray.wait``:
|
||||
|
||||
.. autofunction:: ray.wait
|
||||
:noindex:
|
||||
|
||||
|
||||
@@ -1,14 +1,55 @@
|
||||
"""
|
||||
This file holds code for a Training guide for PytorchSGD in the documentation.
|
||||
|
||||
It ignores yapf because yapf doesn't allow comments right after code blocks,
|
||||
but we put comments right after code blocks to prevent large white spaces
|
||||
in the documentation.
|
||||
"""
|
||||
|
||||
# yapf: disable
|
||||
# __torch_train_example__
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
from ray import tune
|
||||
from ray.experimental.sgd.pytorch.pytorch_trainer import (PyTorchTrainer,
|
||||
PyTorchTrainable)
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from ray.experimental.sgd.tests.pytorch_utils import (
|
||||
model_creator, optimizer_creator, data_creator)
|
||||
from ray.experimental.sgd.pytorch.pytorch_trainer import PyTorchTrainer
|
||||
|
||||
|
||||
class LinearDataset(torch.utils.data.Dataset):
|
||||
"""y = a * x + b"""
|
||||
|
||||
def __init__(self, a, b, size=1000):
|
||||
x = np.random.random(size).astype(np.float32) * 10
|
||||
x = np.arange(0, 10, 10 / size, dtype=np.float32)
|
||||
self.x = torch.from_numpy(x)
|
||||
self.y = torch.from_numpy(a * x + b)
|
||||
|
||||
def __getitem__(self, index):
|
||||
return self.x[index, None], self.y[index, None]
|
||||
|
||||
def __len__(self):
|
||||
return len(self.x)
|
||||
|
||||
|
||||
def model_creator(config):
|
||||
return nn.Linear(1, 1)
|
||||
|
||||
|
||||
def optimizer_creator(model, config):
|
||||
"""Returns criterion, optimizer"""
|
||||
criterion = nn.MSELoss()
|
||||
optimizer = torch.optim.SGD(model.parameters(), lr=1e-4)
|
||||
return criterion, optimizer
|
||||
|
||||
|
||||
def data_creator(config):
|
||||
"""Returns training set, validation set"""
|
||||
return LinearDataset(2, 5), LinearDataset(2, 5, size=400)
|
||||
|
||||
|
||||
def train_example(num_replicas=1, use_gpu=False):
|
||||
@@ -25,27 +66,6 @@ def train_example(num_replicas=1, use_gpu=False):
|
||||
print("success!")
|
||||
|
||||
|
||||
def tune_example(num_replicas=1, use_gpu=False):
|
||||
config = {
|
||||
"model_creator": tune.function(model_creator),
|
||||
"data_creator": tune.function(data_creator),
|
||||
"optimizer_creator": tune.function(optimizer_creator),
|
||||
"num_replicas": num_replicas,
|
||||
"use_gpu": use_gpu,
|
||||
"batch_size": 512,
|
||||
"backend": "gloo"
|
||||
}
|
||||
|
||||
analysis = tune.run(
|
||||
PyTorchTrainable,
|
||||
num_samples=12,
|
||||
config=config,
|
||||
stop={"training_iteration": 2},
|
||||
verbose=1)
|
||||
|
||||
return analysis.get_best_config(metric="validation_loss", mode="min")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
@@ -72,8 +92,4 @@ if __name__ == "__main__":
|
||||
import ray
|
||||
|
||||
ray.init(redis_address=args.redis_address)
|
||||
|
||||
if args.tune:
|
||||
tune_example(num_replicas=args.num_replicas, use_gpu=args.use_gpu)
|
||||
else:
|
||||
train_example(num_replicas=args.num_replicas, use_gpu=args.use_gpu)
|
||||
train_example(num_replicas=args.num_replicas, use_gpu=args.use_gpu)
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
# yapf: disable
|
||||
"""
|
||||
This file holds code for a Distributed Pytorch + Tune page in the docs.
|
||||
|
||||
It ignores yapf because yapf doesn't allow comments right after code blocks,
|
||||
but we put comments right after code blocks to prevent large white spaces
|
||||
in the documentation.
|
||||
"""
|
||||
|
||||
# __torch_tune_example__
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import ray
|
||||
from ray import tune
|
||||
from ray.experimental.sgd.pytorch.pytorch_trainer import PyTorchTrainable
|
||||
|
||||
|
||||
class LinearDataset(torch.utils.data.Dataset):
|
||||
"""y = a * x + b"""
|
||||
|
||||
def __init__(self, a, b, size=1000):
|
||||
x = np.random.random(size).astype(np.float32) * 10
|
||||
x = np.arange(0, 10, 10 / size, dtype=np.float32)
|
||||
self.x = torch.from_numpy(x)
|
||||
self.y = torch.from_numpy(a * x + b)
|
||||
|
||||
def __getitem__(self, index):
|
||||
return self.x[index, None], self.y[index, None]
|
||||
|
||||
def __len__(self):
|
||||
return len(self.x)
|
||||
|
||||
|
||||
def model_creator(config):
|
||||
return nn.Linear(1, 1)
|
||||
|
||||
|
||||
def optimizer_creator(model, config):
|
||||
"""Returns criterion, optimizer"""
|
||||
criterion = nn.MSELoss()
|
||||
optimizer = torch.optim.SGD(model.parameters(), lr=config.get("lr", 1e-4))
|
||||
return criterion, optimizer
|
||||
|
||||
|
||||
def data_creator(config):
|
||||
"""Returns training set, validation set"""
|
||||
return LinearDataset(2, 5), LinearDataset(2, 5, size=400)
|
||||
|
||||
|
||||
def tune_example(num_replicas=1, use_gpu=False):
|
||||
config = {
|
||||
"model_creator": tune.function(model_creator),
|
||||
"data_creator": tune.function(data_creator),
|
||||
"optimizer_creator": tune.function(optimizer_creator),
|
||||
"num_replicas": num_replicas,
|
||||
"use_gpu": use_gpu,
|
||||
"batch_size": 512,
|
||||
"backend": "gloo"
|
||||
}
|
||||
|
||||
analysis = tune.run(
|
||||
PyTorchTrainable,
|
||||
num_samples=12,
|
||||
config=config,
|
||||
stop={"training_iteration": 2},
|
||||
verbose=1)
|
||||
|
||||
return analysis.get_best_config(metric="validation_loss", mode="min")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--redis-address",
|
||||
type=str,
|
||||
help="the address to use for Redis")
|
||||
parser.add_argument(
|
||||
"--num-replicas",
|
||||
"-n",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Sets number of replicas for training.")
|
||||
parser.add_argument(
|
||||
"--use-gpu",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Enables GPU training")
|
||||
parser.add_argument(
|
||||
"--tune", action="store_true", default=False, help="Tune training")
|
||||
|
||||
args, _ = parser.parse_known_args()
|
||||
|
||||
ray.init(redis_address=args.redis_address)
|
||||
tune_example(num_replicas=args.num_replicas, use_gpu=args.use_gpu)
|
||||
@@ -1,4 +1,4 @@
|
||||
Tune: Scalable Hyperparameter Search
|
||||
Tune: Scalable Hyperparameter Tuning
|
||||
====================================
|
||||
|
||||
Tune is a scalable framework for hyperparameter search with a focus on deep learning and deep reinforcement learning.
|
||||
|
||||
Reference in New Issue
Block a user