mirror of
https://github.com/wassname/ray.git
synced 2026-07-01 14:36:45 +08:00
Add a script for benchmarking performance for Ray developers. (#5472)
This commit is contained in:
Robert Nishihara
committed by
Philipp Moritz
Philipp Moritz
parent
99a2f9fab3
commit
851c5b2dae
@@ -1,38 +0,0 @@
|
||||
Running the benchmarks
|
||||
======================
|
||||
|
||||
You can run the benchmark suite by doing the following:
|
||||
|
||||
1. Install https://github.com/ray-project/asv: ``cd asv; pip install -e .``
|
||||
2. Run ``asv dev`` in this directory.
|
||||
|
||||
To run ASV inside docker, you can use the following command:
|
||||
``docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA bash -c '/ray/test/jenkins_tests/run_asv.sh'``
|
||||
``docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA bash -c '/ray/test/jenkins_tests/run_rllib_asv.sh'``
|
||||
|
||||
|
||||
Visualizing Benchmarks
|
||||
======================
|
||||
|
||||
For visualizing regular Ray benchmarks, you must copy the S3 bucket down to `$RAY_DIR/python`.
|
||||
|
||||
.. code-block::
|
||||
|
||||
cd $RAY_DIR/python
|
||||
aws s3 sync s3://$BUCKET/ASV/ .
|
||||
|
||||
For rllib, you must sync a _particular_ folder down to `$RLLIB_DIR (ray/python/ray/rllib)`.
|
||||
|
||||
.. code-block::
|
||||
|
||||
cd $RAY_DIR/python/ray/rllib
|
||||
aws s3 sync s3://$BUCKET/RLLIB_RESULTS/ ./RLLIB_RESULTS
|
||||
|
||||
Then, in the directory, you can run:
|
||||
|
||||
.. code-block::
|
||||
|
||||
asv publish --no-pull
|
||||
asv preview
|
||||
|
||||
This creates the directory and then launches a server at which you can visualize results.
|
||||
@@ -1,85 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import ray
|
||||
|
||||
NUM_WORKERS = 4
|
||||
|
||||
|
||||
def setup():
|
||||
if not hasattr(setup, "is_initialized"):
|
||||
ray.init(num_cpus=4)
|
||||
setup.is_initialized = True
|
||||
|
||||
|
||||
@ray.remote
|
||||
class MyActor(object):
|
||||
def __init__(self):
|
||||
self.x = None
|
||||
|
||||
def get_x(self):
|
||||
return self.x
|
||||
|
||||
def set_x(self, x):
|
||||
self.x = x
|
||||
|
||||
|
||||
class ActorInstantiationSuite(object):
|
||||
def instantiate_actor(self):
|
||||
actor = MyActor.remote()
|
||||
# Block to make sure actor is instantiated
|
||||
ray.get(actor.get_x.remote())
|
||||
|
||||
def instantiate_many_actors(self):
|
||||
actors = [MyActor.remote() for _ in range(NUM_WORKERS + 10)]
|
||||
ray.get([actor.get_x.remote() for actor in actors])
|
||||
|
||||
def time_instantiate_actor(self):
|
||||
self.instantiate_actor()
|
||||
|
||||
def peakmem_instantiate_actor(self):
|
||||
self.instantiate_actor()
|
||||
|
||||
def time_instantiate_many_actors(self):
|
||||
self.instantiate_many_actors()
|
||||
|
||||
def peakmem_instantiate_many_actors(self):
|
||||
self.instantiate_many_actors()
|
||||
|
||||
|
||||
class ActorMethodSuite(object):
|
||||
def setup(self):
|
||||
self.actor = MyActor.remote()
|
||||
# Block to make sure actor is instantiated
|
||||
ray.get(self.actor.get_x.remote())
|
||||
|
||||
def time_call_method(self):
|
||||
ray.get(self.actor.get_x.remote())
|
||||
|
||||
def peakmem_call_method(self):
|
||||
ray.get(self.actor.get_x.remote())
|
||||
|
||||
|
||||
class ActorCheckpointSuite(object):
|
||||
def checkpoint_and_restore(self):
|
||||
actor = MyActor.remote()
|
||||
actor.__ray_checkpoint__.remote()
|
||||
assert ray.get(actor.__ray_checkpoint_restore__.remote())
|
||||
|
||||
def save_checkpoint(self):
|
||||
actor = MyActor.remote()
|
||||
checkpoint = ray.get(actor.__ray_save_checkpoint__.remote())
|
||||
return checkpoint
|
||||
|
||||
def time_checkpoint_and_restore(self):
|
||||
self.checkpoint_and_restore()
|
||||
|
||||
def peakmem_checkpoint_and_restore(self):
|
||||
self.checkpoint_and_restore()
|
||||
|
||||
def time_save_checkpoint(self):
|
||||
self.save_checkpoint()
|
||||
|
||||
def mem_save_checkpoint(self):
|
||||
return self.save_checkpoint()
|
||||
@@ -1,108 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import numpy as np
|
||||
|
||||
import ray
|
||||
|
||||
|
||||
def setup():
|
||||
if not hasattr(setup, "is_initialized"):
|
||||
ray.init(num_cpus=4)
|
||||
setup.is_initialized = True
|
||||
|
||||
|
||||
def square(x):
|
||||
return x * x
|
||||
|
||||
|
||||
class Foo(object):
|
||||
def bar(self):
|
||||
return 42
|
||||
|
||||
|
||||
class GetBase(object):
|
||||
def setup(self):
|
||||
self.oid = ray.put(None)
|
||||
|
||||
def time_get(self):
|
||||
ray.get(self.oid)
|
||||
|
||||
def peakmem_get(self):
|
||||
ray.get(self.oid)
|
||||
|
||||
|
||||
class GetBoolSuite(GetBase):
|
||||
def setup(self):
|
||||
self.oid = ray.put(True)
|
||||
|
||||
|
||||
class GetIntSuite(GetBase):
|
||||
def setup(self):
|
||||
self.oid = ray.put(42)
|
||||
|
||||
|
||||
class GetFloatSuite(GetBase):
|
||||
def setup(self):
|
||||
self.oid = ray.put(4.2)
|
||||
|
||||
|
||||
class GetComplexSuite(GetBase):
|
||||
def setup(self):
|
||||
self.oid = ray.put(4 + 2j)
|
||||
|
||||
|
||||
class GetNoneSuite(GetBase):
|
||||
def setup(self):
|
||||
self.oid = ray.put(None)
|
||||
|
||||
|
||||
class GetStringSuite(GetBase):
|
||||
def setup(self):
|
||||
self.oid = ray.put("forty-two")
|
||||
|
||||
|
||||
class GetBytesSuite(GetBase):
|
||||
def setup(self):
|
||||
self.oid = ray.put(b"forty-two")
|
||||
|
||||
|
||||
class GetListSuite(GetBase):
|
||||
def setup(self):
|
||||
self.oid = ray.put([i for i in range(100)])
|
||||
|
||||
|
||||
class GetSetSuite(GetBase):
|
||||
def setup(self):
|
||||
self.oid = ray.put({i for i in range(100)})
|
||||
|
||||
|
||||
class GetTupleSuite(GetBase):
|
||||
def setup(self):
|
||||
self.oid = ray.put(tuple(range(100)))
|
||||
|
||||
|
||||
class GetDictSuite(GetBase):
|
||||
def setup(self):
|
||||
self.oid = ray.put({i: i for i in range(100)})
|
||||
|
||||
|
||||
class GetFunctionSuite(GetBase):
|
||||
def setup(self):
|
||||
self.oid = ray.put(square)
|
||||
|
||||
|
||||
class GetClassSuite(GetBase):
|
||||
def setup(self):
|
||||
self.oid = ray.put(Foo)
|
||||
|
||||
|
||||
class GetClassInstanceSuite(GetBase):
|
||||
def setup(self):
|
||||
self.oid = ray.put(Foo())
|
||||
|
||||
|
||||
class GetArraySuite(GetBase):
|
||||
def setup(self):
|
||||
self.oid = ray.put(np.random.random((100, 100, 100)))
|
||||
@@ -1,108 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import numpy as np
|
||||
|
||||
import ray
|
||||
|
||||
|
||||
def setup():
|
||||
if not hasattr(setup, "is_initialized"):
|
||||
ray.init(num_cpus=0)
|
||||
setup.is_initialized = True
|
||||
|
||||
|
||||
def square(x):
|
||||
return x * x
|
||||
|
||||
|
||||
class Foo(object):
|
||||
def bar(self):
|
||||
return 42
|
||||
|
||||
|
||||
class PutBase(object):
|
||||
def setup(self):
|
||||
self.object = None
|
||||
|
||||
def time_put(self):
|
||||
ray.put(self.object)
|
||||
|
||||
def peakmem_put(self):
|
||||
ray.put(self.object)
|
||||
|
||||
|
||||
class PutBoolSuite(PutBase):
|
||||
def setup(self):
|
||||
self.object = True
|
||||
|
||||
|
||||
class PutIntSuite(PutBase):
|
||||
def setup(self):
|
||||
self.object = 42
|
||||
|
||||
|
||||
class PutFloatSuite(PutBase):
|
||||
def setup(self):
|
||||
self.object = 4.2
|
||||
|
||||
|
||||
class PutComplexSuite(PutBase):
|
||||
def setup(self):
|
||||
self.object = 4 + 2j
|
||||
|
||||
|
||||
class PutNoneSuite(PutBase):
|
||||
def setup(self):
|
||||
self.object = None
|
||||
|
||||
|
||||
class PutStringSuite(PutBase):
|
||||
def setup(self):
|
||||
self.object = "forty-two"
|
||||
|
||||
|
||||
class PutBytesSuite(PutBase):
|
||||
def setup(self):
|
||||
self.object = b"forty-two"
|
||||
|
||||
|
||||
class PutListSuite(PutBase):
|
||||
def setup(self):
|
||||
self.object = [i for i in range(100)]
|
||||
|
||||
|
||||
class PutSetSuite(PutBase):
|
||||
def setup(self):
|
||||
self.object = {i for i in range(100)}
|
||||
|
||||
|
||||
class PutTupleSuite(PutBase):
|
||||
def setup(self):
|
||||
self.object = tuple(range(100))
|
||||
|
||||
|
||||
class PutDictSuite(PutBase):
|
||||
def setup(self):
|
||||
self.object = {i: i for i in range(100)}
|
||||
|
||||
|
||||
class PutFunctionSuite(PutBase):
|
||||
def setup(self):
|
||||
self.object = square
|
||||
|
||||
|
||||
class PutClassSuite(PutBase):
|
||||
def setup(self):
|
||||
self.object = Foo
|
||||
|
||||
|
||||
class PutClassInstanceSuite(PutBase):
|
||||
def setup(self):
|
||||
self.object = Foo()
|
||||
|
||||
|
||||
class PutArraySuite(PutBase):
|
||||
def setup(self):
|
||||
self.object = np.random.random((100, 100, 100))
|
||||
@@ -1,31 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import ray
|
||||
from ray.experimental.queue import Queue
|
||||
|
||||
|
||||
def setup():
|
||||
if not hasattr(setup, "is_initialized"):
|
||||
ray.init(num_cpus=4)
|
||||
setup.is_initialized = True
|
||||
|
||||
|
||||
class QueueSuite(object):
|
||||
def time_put(self):
|
||||
queue = Queue(1000)
|
||||
for i in range(1000):
|
||||
queue.put(i)
|
||||
|
||||
def time_get(self):
|
||||
queue = Queue()
|
||||
for i in range(1000):
|
||||
queue.put(i)
|
||||
for _ in range(1000):
|
||||
queue.get()
|
||||
|
||||
def time_qsize(self):
|
||||
queue = Queue()
|
||||
for _ in range(1000):
|
||||
queue.qsize()
|
||||
@@ -1,61 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import ray
|
||||
|
||||
|
||||
def setup():
|
||||
if not hasattr(setup, "is_initialized"):
|
||||
ray.init(num_cpus=10, resources={"foo": 1})
|
||||
setup.is_initialized = True
|
||||
|
||||
|
||||
def square(x):
|
||||
return x * x
|
||||
|
||||
|
||||
class TaskSuite(object):
|
||||
timeout = 10
|
||||
|
||||
def setup(self):
|
||||
self.square = ray.remote(square)
|
||||
|
||||
def run_many_tasks(self):
|
||||
ray.get([self.square.remote(i) for i in range(100)])
|
||||
|
||||
def run_task_dependency(self):
|
||||
first_oid = self.square.remote(2)
|
||||
second_oid = self.square.remote(first_oid)
|
||||
ray.get(second_oid)
|
||||
|
||||
def time_submit_task(self):
|
||||
self.square.remote(2)
|
||||
|
||||
def time_task_lifecycle(self):
|
||||
ray.get(self.square.remote(2))
|
||||
|
||||
def peakmem_task_lifecycle(self):
|
||||
ray.get(self.square.remote(2))
|
||||
|
||||
def time_run_many_tasks(self):
|
||||
self.run_many_tasks()
|
||||
|
||||
def peakmem_run_many_tasks(self):
|
||||
self.run_many_tasks()
|
||||
|
||||
def time_task_dependency(self):
|
||||
self.run_task_dependency()
|
||||
|
||||
def peakmem_task_dependency(self):
|
||||
self.run_task_dependency()
|
||||
|
||||
|
||||
class CPUTaskSuite(TaskSuite):
|
||||
def setup(self):
|
||||
self.square = ray.remote(num_cpus=1)(square)
|
||||
|
||||
|
||||
class CustomResourceTaskSuite(TaskSuite):
|
||||
def setup(self):
|
||||
self.square = ray.remote(resources={"foo": 1})(square)
|
||||
@@ -1,39 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import time
|
||||
|
||||
import ray
|
||||
|
||||
|
||||
def setup(*args):
|
||||
if not hasattr(setup, "is_initialized"):
|
||||
ray.init(num_cpus=4)
|
||||
setup.is_initialized = True
|
||||
|
||||
|
||||
@ray.remote
|
||||
def sleep(x):
|
||||
time.sleep(x)
|
||||
|
||||
|
||||
class WaitSuite(object):
|
||||
timeout = 0.01
|
||||
timer = time.time
|
||||
|
||||
def time_wait_task(self):
|
||||
ray.wait([sleep.remote(0.1)])
|
||||
|
||||
def time_wait_many_tasks(self, num_returns):
|
||||
tasks = [sleep.remote(i / 5) for i in range(4)]
|
||||
ray.wait(tasks, num_returns=num_returns)
|
||||
|
||||
time_wait_many_tasks.params = list(range(1, 4))
|
||||
time_wait_many_tasks.param_names = ["num_returns"]
|
||||
|
||||
def time_wait_timeout(self, timeout):
|
||||
ray.wait([sleep.remote(0.5)], timeout=timeout)
|
||||
|
||||
time_wait_timeout.params = [0.2, 0.8]
|
||||
time_wait_timeout.param_names = ["timeout"]
|
||||
@@ -1,51 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import ray
|
||||
|
||||
|
||||
def setup():
|
||||
if not hasattr(setup, "is_initialized"):
|
||||
ray.init(num_cpus=4)
|
||||
setup.is_initialized = True
|
||||
|
||||
|
||||
@ray.remote
|
||||
def trivial_function():
|
||||
return 1
|
||||
|
||||
|
||||
class TimeSuite(object):
|
||||
"""An example benchmark."""
|
||||
|
||||
def setup(self):
|
||||
self.d = {}
|
||||
for x in range(500):
|
||||
self.d[x] = None
|
||||
|
||||
def time_keys(self):
|
||||
for key in self.d.keys():
|
||||
pass
|
||||
|
||||
def time_range(self):
|
||||
d = self.d
|
||||
for key in range(500):
|
||||
d[key]
|
||||
|
||||
|
||||
class MemSuite(object):
|
||||
def mem_list(self):
|
||||
return [0] * 256
|
||||
|
||||
|
||||
class MicroBenchmarkSuite(object):
|
||||
def time_submit(self):
|
||||
trivial_function.remote()
|
||||
|
||||
def time_submit_and_get(self):
|
||||
x = trivial_function.remote()
|
||||
ray.get(x)
|
||||
|
||||
def time_put(self):
|
||||
ray.put(1)
|
||||
Reference in New Issue
Block a user