From a3181816b3e1d08265663dd3d9a46af919ae3903 Mon Sep 17 00:00:00 2001 From: Simon Mo Date: Thu, 2 Apr 2020 15:31:03 -0700 Subject: [PATCH] Add result from 0.8.4 release (#7876) --- doc/dev/download_wheels.sh | 2 + doc/dev/release_logs/0.8.4/microbenchmark.txt | 14 ++++++ .../release_logs/0.8.4/rllib_regression.txt | 47 +++++++++++++++++++ .../stress_tests/application_stress_test.txt | 14 ++++++ .../0.8.4/stress_tests/test_dead_actors.txt | 4 ++ .../0.8.4/stress_tests/test_many_tasks.txt | 15 ++++++ 6 files changed, 96 insertions(+) create mode 100644 doc/dev/release_logs/0.8.4/microbenchmark.txt create mode 100644 doc/dev/release_logs/0.8.4/rllib_regression.txt create mode 100644 doc/dev/release_logs/0.8.4/stress_tests/application_stress_test.txt create mode 100644 doc/dev/release_logs/0.8.4/stress_tests/test_dead_actors.txt create mode 100644 doc/dev/release_logs/0.8.4/stress_tests/test_many_tasks.txt diff --git a/doc/dev/download_wheels.sh b/doc/dev/download_wheels.sh index ad1a3b456..011d64bf2 100644 --- a/doc/dev/download_wheels.sh +++ b/doc/dev/download_wheels.sh @@ -1,6 +1,8 @@ wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp35-cp35m-manylinux1_x86_64.whl wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-manylinux1_x86_64.whl wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-manylinux1_x86_64.whl +wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-manylinux1_x86_64.whl wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp35-cp35m-macosx_10_13_intel.whl wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-macosx_10_13_intel.whl wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-macosx_10_13_intel.whl +wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-macosx_10_13_x86_64.whl diff --git a/doc/dev/release_logs/0.8.4/microbenchmark.txt b/doc/dev/release_logs/0.8.4/microbenchmark.txt new file mode 100644 index 000000000..98fe32ccf --- /dev/null +++ b/doc/dev/release_logs/0.8.4/microbenchmark.txt @@ -0,0 +1,14 @@ +single client get calls per second 9887.7 +- 1181.06 +single client put calls per second 5061.57 +- 89.58 +single client put gigabytes per second 13.23 +- 2.84 +multi client put calls per second 10954.66 +- 107.58 +multi client put gigabytes per second 17.98 +- 8.51 +single client tasks sync per second 1246.16 +- 64.56 +single client tasks async per second 12836.66 +- 196.27 +multi client tasks async per second 31252.17 +- 9348.4 +1:1 actor calls sync per second 1974.26 +- 31.31 +1:1 actor calls async per second 7211.89 +- 158.3 +1:1 actor calls concurrent per second 6554.62 +- 140.58 +1:n actor calls async per second 9934.44 +- 103.02 +n:n actor calls async per second 34703.22 +- 553.23 +n:n actor calls with arg async per second 12569.31 +- 257.01 diff --git a/doc/dev/release_logs/0.8.4/rllib_regression.txt b/doc/dev/release_logs/0.8.4/rllib_regression.txt new file mode 100644 index 000000000..7f2882688 --- /dev/null +++ b/doc/dev/release_logs/0.8.4/rllib_regression.txt @@ -0,0 +1,47 @@ +== Status == +Memory usage on this node: 166.1/480.3 GiB +Using FIFO scheduling algorithm. +Resources requested: 0/64 CPUs, 0.0/8 GPUs, 0.0/323.54 GiB heap, 0.0/98.39 GiB objects +Result logdir: /home/ubuntu/ray_results/apex +Result logdir: /home/ubuntu/ray_results/atari-a2c +Result logdir: /home/ubuntu/ray_results/atari-basic-dqn +Result logdir: /home/ubuntu/ray_results/atari-impala +Result logdir: /home/ubuntu/ray_results/atari-ppo-tf +Result logdir: /home/ubuntu/ray_results/atari-ppo-torch +Number of trials: 24 (8 ERROR, 16 TERMINATED) ++-------------------------------------+------------+-------+--------+------------------+---------+-----------+ +| Trial name | status | loc | iter | total time (s) | ts | reward | +|-------------------------------------+------------+-------+--------+------------------+---------+-----------| +| IMPALA_BreakoutNoFrameskip-v4_00000 | TERMINATED | | 293 | 3612.08 | 7270500 | 351.08 | +| IMPALA_BreakoutNoFrameskip-v4_00001 | TERMINATED | | 293 | 3612.11 | 7252500 | 181.3 | +| IMPALA_BreakoutNoFrameskip-v4_00002 | TERMINATED | | 293 | 3610.22 | 7283000 | 377.12 | +| IMPALA_BreakoutNoFrameskip-v4_00003 | TERMINATED | | 293 | 3612.02 | 7238000 | 1.9927 | +| PPO_BreakoutNoFrameskip-v4_00004 | TERMINATED | | 680 | 3602.41 | 3400000 | 45.13 | +| PPO_BreakoutNoFrameskip-v4_00005 | TERMINATED | | 853 | 3600.28 | 4265000 | 45.87 | +| PPO_BreakoutNoFrameskip-v4_00006 | TERMINATED | | 863 | 3603.41 | 4315000 | 40.92 | +| PPO_BreakoutNoFrameskip-v4_00007 | TERMINATED | | 859 | 3600.36 | 4295000 | 80.42 | +| PPO_BreakoutNoFrameskip-v4_00000 | TERMINATED | | 433 | 3606.17 | 2165000 | 23.65 | +| PPO_BreakoutNoFrameskip-v4_00001 | TERMINATED | | 435 | 3606.03 | 2175000 | 18.35 | +| PPO_BreakoutNoFrameskip-v4_00002 | TERMINATED | | 436 | 3608.19 | 2180000 | 31.09 | +| PPO_BreakoutNoFrameskip-v4_00003 | TERMINATED | | 434 | 3602.58 | 2170000 | 17.32 | +| APEX_BreakoutNoFrameskip-v4_00004 | TERMINATED | | 115 | 3609.98 | 6512320 | 55.76 | +| APEX_BreakoutNoFrameskip-v4_00005 | TERMINATED | | 115 | 3609.56 | 6491040 | 66.91 | +| APEX_BreakoutNoFrameskip-v4_00006 | TERMINATED | | 116 | 3607.5 | 8302720 | 91.65 | +| APEX_BreakoutNoFrameskip-v4_00007 | TERMINATED | | 116 | 3605.31 | 8500960 | 79.94 | +| A2C_BreakoutNoFrameskip-v4_00016 | TERMINATED | | 354 | 3608.77 | 4223000 | 288.79 | +| A2C_BreakoutNoFrameskip-v4_00017 | TERMINATED | | 354 | 3608.4 | 4119000 | 231.83 | +| A2C_BreakoutNoFrameskip-v4_00018 | TERMINATED | | 354 | 3605.56 | 4121500 | 169.97 | +| A2C_BreakoutNoFrameskip-v4_00019 | TERMINATED | | 354 | 3606.65 | 4169500 | 276.07 | +| DQN_BreakoutNoFrameskip-v4_00020 | TERMINATED | | 33 | 3675.38 | 330000 | 15.44 | +| DQN_BreakoutNoFrameskip-v4_00021 | TERMINATED | | 33 | 3674.96 | 330000 | 13.14 | +| DQN_BreakoutNoFrameskip-v4_00022 | TERMINATED | | 33 | 3679.57 | 330000 | 17.47 | +| DQN_BreakoutNoFrameskip-v4_00023 | TERMINATED | | 33 | 3657.31 | 330000 | 17.47 | ++-------------------------------------+------------+-------+--------+------------------+---------+-----------+ + + + + + + + + diff --git a/doc/dev/release_logs/0.8.4/stress_tests/application_stress_test.txt b/doc/dev/release_logs/0.8.4/stress_tests/application_stress_test.txt new file mode 100644 index 000000000..efc040107 --- /dev/null +++ b/doc/dev/release_logs/0.8.4/stress_tests/application_stress_test.txt @@ -0,0 +1,14 @@ +== Status == +Memory usage on this node: 24.9/480.3 GiB +Using FIFO scheduling algorithm. +Resources requested: 0/640 CPUs, 0/8 GPUs, 0.0/1905.71 GiB heap, 0.0/566.21 GiB objects +Result logdir: /home/ubuntu/ray_results/atari-impala +Number of trials: 4 (4 TERMINATED) ++------------------------------------------+------------+-------+-----------------------------+--------+------------------+----------+----------+ +| Trial name | status | loc | env | iter | total time (s) | ts | reward | +|------------------------------------------+------------+-------+-----------------------------+--------+------------------+----------+----------| +| IMPALA_BreakoutNoFrameskip-v4_00000 | TERMINATED | | BreakoutNoFrameskip-v4 | 379 | 6695.67 | 30028000 | 547.94 | +| IMPALA_BeamRiderNoFrameskip-v4_00001 | TERMINATED | | BeamRiderNoFrameskip-v4 | 382 | 6742.5 | 30068000 | 2520.02 | +| IMPALA_QbertNoFrameskip-v4_00002 | TERMINATED | | QbertNoFrameskip-v4 | 374 | 6623.08 | 30024500 | 9815 | +| IMPALA_SpaceInvadersNoFrameskip-v4_00003 | TERMINATED | | SpaceInvadersNoFrameskip-v4 | 380 | 6711.43 | 30058500 | 762.9 | ++------------------------------------------+------------+-------+-----------------------------+--------+------------------+----------+----------+ diff --git a/doc/dev/release_logs/0.8.4/stress_tests/test_dead_actors.txt b/doc/dev/release_logs/0.8.4/stress_tests/test_dead_actors.txt new file mode 100644 index 000000000..e0ef8ef1e --- /dev/null +++ b/doc/dev/release_logs/0.8.4/stress_tests/test_dead_actors.txt @@ -0,0 +1,4 @@ +Finished in: 103.75814247131348s +Average iteration time: 1.037579414844513s +Max iteration time: 2.213489055633545s +Min iteration time: 0.09523892402648926s diff --git a/doc/dev/release_logs/0.8.4/stress_tests/test_many_tasks.txt b/doc/dev/release_logs/0.8.4/stress_tests/test_many_tasks.txt new file mode 100644 index 000000000..e1c5a542f --- /dev/null +++ b/doc/dev/release_logs/0.8.4/stress_tests/test_many_tasks.txt @@ -0,0 +1,15 @@ +Stage 0 results: + Total time: 24.571804523468018 +Stage 1 results: + Total time: 155.27635407447815 + Average iteration time: 15.527625465393067 + Max iteration time: 16.342751502990723 + Min iteration time: 15.177084684371948 +Stage 2 results: + Total time: 661.6063549518585 + Average iteration time: 132.3208240032196 + Max iteration time: 134.2343475818634 + Min iteration time: 128.90119433403015 +Stage 3 results: + Actor creation time: 0.07286763191223145 + Total time: 3591.7382633686066