From 6725a61bdafb7d34922604e0f95348c14d8175bb Mon Sep 17 00:00:00 2001 From: Eric Liang Date: Tue, 17 Dec 2019 15:56:50 -0800 Subject: [PATCH] Release 0.8.0 test logs (#6512) --- ci/stress_tests/test_many_tasks.py | 17 +++++++++++++ doc/dev/RELEASE_PROCESS.rst | 5 ++-- doc/dev/release_logs/0.8.0/microbenchmark.txt | 14 +++++++++++ .../release_logs/0.8.0/rllib_regression.txt | 24 +++++++++++++++++++ .../0.8.0/stress_tests/test_dead_actors.txt | 4 ++++ .../0.8.0/stress_tests/test_many_tasks.txt | 15 ++++++++++++ doc/source/profiling.rst | 20 +--------------- 7 files changed, 77 insertions(+), 22 deletions(-) create mode 100644 doc/dev/release_logs/0.8.0/microbenchmark.txt create mode 100644 doc/dev/release_logs/0.8.0/rllib_regression.txt create mode 100644 doc/dev/release_logs/0.8.0/stress_tests/test_dead_actors.txt create mode 100644 doc/dev/release_logs/0.8.0/stress_tests/test_many_tasks.txt diff --git a/ci/stress_tests/test_many_tasks.py b/ci/stress_tests/test_many_tasks.py index 8d9777a71..c62031b37 100644 --- a/ci/stress_tests/test_many_tasks.py +++ b/ci/stress_tests/test_many_tasks.py @@ -47,6 +47,20 @@ class Actor(object): return np.ones(size, dtype=np.uint8) +# Stage 0: Submit a bunch of small tasks with large returns. +stage_0_iterations = [] +start_time = time.time() +logger.info("Submitting many tasks with large returns.") +for i in range(10): + iteration_start = time.time() + logger.info("Iteration %s", i) + ray.get([f.remote(1000000) for _ in range(1000)]) + stage_0_iterations.append(time.time() - iteration_start) + +stage_0_time = time.time() - start_time +logger.info("Finished stage 0 after %s seconds.", stage_0_time) + + # Stage 1: Launch a bunch of tasks. stage_1_iterations = [] start_time = time.time() @@ -101,6 +115,9 @@ for N in [1000, 100000]: stage_3_time = time.time() - start_time logger.info("Finished stage 3 in %s seconds.", stage_3_time) +print("Stage 0 results:") +print("\tTotal time: {}".format(stage_0_time)) + print("Stage 1 results:") print("\tTotal time: {}".format(stage_1_time)) print("\tAverage iteration time: {}".format( diff --git a/doc/dev/RELEASE_PROCESS.rst b/doc/dev/RELEASE_PROCESS.rst index a6c6103b5..7980b061e 100644 --- a/doc/dev/RELEASE_PROCESS.rst +++ b/doc/dev/RELEASE_PROCESS.rst @@ -64,10 +64,9 @@ This document describes the process for creating new releases. ray microbenchmark Run `ray microbenchmark` on an `m4.16xl` instance running `Ubuntu 18.04` with `Python 3` to get the - latest microbenchmark numbers and update them in `profiling.rst`. + latest microbenchmark numbers. - The results should be updated in ``doc/dev/profiling.rst`` and checked in under - ``doc/dev/release_logs/``. + The results should be checked in under ``doc/dev/release_logs/``. 5. **Resolve release-blockers:** If a release blocking issue arises, there are two ways the issue can be resolved: 1) Fix the issue on the master branch and diff --git a/doc/dev/release_logs/0.8.0/microbenchmark.txt b/doc/dev/release_logs/0.8.0/microbenchmark.txt new file mode 100644 index 000000000..9406ee1af --- /dev/null +++ b/doc/dev/release_logs/0.8.0/microbenchmark.txt @@ -0,0 +1,14 @@ +single client get calls per second 30526.52 +- 1121.22 +single client put calls per second 6428.11 +- 51.25 +single client put gigabytes per second 12.85 +- 5.45 +multi client put calls per second 16976.5 +- 60.13 +multi client put gigabytes per second 22.54 +- 0.86 +single client tasks sync per second 1223.63 +- 18.36 +single client tasks async per second 13715.93 +- 229.68 +multi client tasks async per second 38344.2 +- 4336.41 +1:1 actor calls sync per second 2318.72 +- 24.19 +1:1 actor calls async per second 7530.01 +- 222.72 +1:1 actor calls concurrent per second 6981.48 +- 228.29 +1:n actor calls async per second 13974.54 +- 41.26 +n:n actor calls async per second 43835.4 +- 400.04 +n:n actor calls with arg async per second 14775.69 +- 295.24 diff --git a/doc/dev/release_logs/0.8.0/rllib_regression.txt b/doc/dev/release_logs/0.8.0/rllib_regression.txt new file mode 100644 index 000000000..14fc65e2a --- /dev/null +++ b/doc/dev/release_logs/0.8.0/rllib_regression.txt @@ -0,0 +1,24 @@ ++----------------------------------------+------------+--------------------+--------+------------------+-------------+----------+ +| Trial name | status | loc | iter | total time (s) | timesteps | reward | +|----------------------------------------+------------+--------------------+--------+------------------+-------------+----------| +| IMPALA_BreakoutNoFrameskip-v4_e25c5ce8 | TERMINATED | | 310 | 3604.3 | 9648000 | 399.75 | +| IMPALA_BreakoutNoFrameskip-v4_e25cdfec | TERMINATED | | 310 | 3611.22 | 9552000 | 403.49 | +| IMPALA_BreakoutNoFrameskip-v4_e25d65de | TERMINATED | | 310 | 3604.05 | 9604500 | 390.47 | +| IMPALA_BreakoutNoFrameskip-v4_e25ddf0a | TERMINATED | | 310 | 3604.18 | 9447000 | 365.57 | +| PPO_BreakoutNoFrameskip-v4_e25e4ddc | TERMINATED | | 604 | 3605.24 | 3020000 | 42.59 | +| PPO_BreakoutNoFrameskip-v4_e25eb8da | TERMINATED | | 613 | 3603.35 | 3065000 | 50.58 | +| PPO_BreakoutNoFrameskip-v4_e25f4520 | TERMINATED | | 608 | 3600.01 | 3040000 | 51.2 | +| PPO_BreakoutNoFrameskip-v4_e25fd44a | TERMINATED | | 613 | 3604.97 | 3065000 | 26.19 | +| APEX_BreakoutNoFrameskip-v4_e2603fa2 | TERMINATED | | 115 | 3622.22 | 5169280 | 48.56 | +| APEX_BreakoutNoFrameskip-v4_e260d462 | TERMINATED | | 115 | 3616.47 | 5201280 | 45.01 | +| APEX_BreakoutNoFrameskip-v4_e2615a90 | TERMINATED | | 115 | 3615.15 | 5134080 | 25.02 | +| APEX_BreakoutNoFrameskip-v4_e261cbec | TERMINATED | | 115 | 3625.55 | 5189280 | 47.01 | +| A2C_BreakoutNoFrameskip-v4_e26260a2 | TERMINATED | | 351 | 3602.43 | 3478000 | 145.84 | +| A2C_BreakoutNoFrameskip-v4_e26331f8 | TERMINATED | | 351 | 3601.93 | 3577000 | 178.73 | +| A2C_BreakoutNoFrameskip-v4_e2639e68 | TERMINATED | | 351 | 3607.28 | 3521500 | 185.45 | +| A2C_BreakoutNoFrameskip-v4_e2642630 | TERMINATED | | 351 | 3606.35 | 3460000 | 122.32 | +| DQN_BreakoutNoFrameskip-v4_99c588f2 | TERMINATED | | 32 | 3643.26 | 320000 | 19.24 | +| DQN_BreakoutNoFrameskip-v4_99c62de8 | TERMINATED | | 31 | 3619.43 | 310000 | 19.97 | +| DQN_BreakoutNoFrameskip-v4_99c6c01e | TERMINATED | | 33 | 3680.95 | 330000 | 16.91 | +| DQN_BreakoutNoFrameskip-v4_99c73f44 | TERMINATED | | 31 | 3625.12 | 310000 | 16.23 | ++----------------------------------------+------------+--------------------+--------+------------------+-------------+----------+ diff --git a/doc/dev/release_logs/0.8.0/stress_tests/test_dead_actors.txt b/doc/dev/release_logs/0.8.0/stress_tests/test_dead_actors.txt new file mode 100644 index 000000000..e1a23032a --- /dev/null +++ b/doc/dev/release_logs/0.8.0/stress_tests/test_dead_actors.txt @@ -0,0 +1,4 @@ +Finished in: 220.28446006774902s +Average iteration time: 2.2028422021865843s +Max iteration time: 4.1706767082214355s +Min iteration time: 1.1767382621765137s diff --git a/doc/dev/release_logs/0.8.0/stress_tests/test_many_tasks.txt b/doc/dev/release_logs/0.8.0/stress_tests/test_many_tasks.txt new file mode 100644 index 000000000..c475dae9b --- /dev/null +++ b/doc/dev/release_logs/0.8.0/stress_tests/test_many_tasks.txt @@ -0,0 +1,15 @@ +Stage 0 results: + Total time: 18.916658878326416 +Stage 1 results: + Total time: 112.98085951805115 + Average iteration time: 11.298071336746215 + Max iteration time: 11.726166009902954 + Min iteration time: 11.037099123001099 +Stage 2 results: + Total time: 1492.7644929885864 + Average iteration time: 298.5524812221527 + Max iteration time: 334.69453477859497 + Min iteration time: 233.40272068977356 +Stage 3 results: + Actor creation time: 0.03908824920654297 + Total time: 2432.014177799225 diff --git a/doc/source/profiling.rst b/doc/source/profiling.rst index c784804aa..364adffcb 100644 --- a/doc/source/profiling.rst +++ b/doc/source/profiling.rst @@ -61,25 +61,7 @@ To run a set of single-node Ray microbenchmarks, use: ray microbenchmark -The following are the results for the 0.7.6 release on a m4.16xl instance running -Ubuntu 18.04 and Python 3.6: - -.. code-block:: text - - single client get calls per second 28595.02 +- 580.33 - single client put calls per second 6313.62 +- 66.88 - single client put gigabytes per second 11.6 +- 6.86 - multi client put calls per second 16800.89 +- 381.69 - multi client put gigabytes per second 23.33 +- 0.96 - single client tasks sync per second 1963.72 +- 48.48 - single client tasks async per second 5181.29 +- 30.0 - multi client tasks async per second 5566.7 +- 280.72 - 1:1 actor calls sync per second 1595.47 +- 38.32 - 1:1 actor calls async per second 2496.26 +- 37.62 - 1:1 direct actor calls async per second 7233.63 +- 205.75 - n:n actor calls async per second 5357.63 +- 116.9 - n:n direct actor calls async per second 90703.32 +- 805.56 - n:n direct actor calls with arg async per second 13300.47 +- 532.66 +You can find the microbenchmark results for Ray releases in the `GitHub release logs `__. References ----------