0.8.5 Release change. (#8358)

ci/long_running_tests/README.rst

@@ -9,6 +9,7 @@ forever until they fail. To set up the project you need to run
     pip install anyscale
     anyscale project create
 
+Note that all the long running test is running inside virtual environment, tensorflow_p36
 
 Running the Workloads
 ---------------------

ci/long_running_tests/ray-project/cluster.yaml

@@ -36,6 +36,7 @@ worker_nodes:
 setup_commands:
     # Install latest TensorFlow
     - source activate tensorflow_p36 && conda remove -y --force wrapt || true
+    - source activate tensorflow_p36 && pip install --upgrade pip
     - source activate tensorflow_p36 && pip install -U tensorflow==1.14
     - echo set-window-option -g mouse on > ~/.tmux.conf
     - echo 'termcapinfo xterm* ti@:te@' > ~/.screenrc

ci/microbenchmark/ray-project/cluster.yaml

@@ -44,6 +44,7 @@ head_setup_commands:
     - wget --quiet https://repo.continuum.io/archive/Anaconda3-5.0.1-Linux-x86_64.sh || true
     - bash Anaconda3-5.0.1-Linux-x86_64.sh -b -p $HOME/anaconda3 || true
     - echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc
+    - pip install -U pip 
 
 # Custom commands that will be run on worker nodes after common setup.
 worker_setup_commands: []

ci/regression_test/rllib_regresssion_tests/ray-project/project.yaml

@@ -38,7 +38,9 @@ commands:
     help: "Show tail of the workoad output."
 
   - name: run-regression-tests
-    command: source activate tensorflow_p36 && rllib train -f compact-regression-test.yaml
+    command: |
+      source activate tensorflow_p36 && pip install torch torchvision
+      source activate tensorflow_p36 && rllib train -f compact-regression-test.yaml
     help: "Run rllib regression tests"
 
 # Pathnames for files and directories that should be saved

ci/regression_test/stress_tests/ray-project/cluster.yaml

@@ -96,6 +96,7 @@ setup_commands:
     # # Build Ray.
     # - git clone https://github.com/ray-project/ray || true
     # - ray/ci/travis/install-bazel.sh
+    - pip install -U pip 
     - pip install boto3==1.4.8 cython==0.29.0
     # - cd ray/python; git checkout master; git pull; pip install -e . --verbose
     - "pip install https://s3-us-west-2.amazonaws.com/ray-wheels/{{ray_branch}}/{{commit}}/ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl"

doc/dev/RELEASE_PROCESS.rst

@@ -66,6 +66,9 @@ This document describes the process for creating new releases.
 
    The results should be checked in under ``doc/dev/release_logs/<version>``.
 
+   You can also get the performance change rate from the previous version using 
+   microbenchmark_analysis.py
+
 5. **Resolve release-blockers:** If a release blocking issue arises, there are
    two ways the issue can be resolved: 1) Fix the issue on the master branch and
    cherry-pick the relevant commit  (using ``git cherry-pick``) onto the release
@@ -117,6 +120,8 @@ This document describes the process for creating new releases.
        pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-macosx_10_13_intel.whl
        pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-macosx_10_13_intel.whl
 
+   This can be tested if you use the script source ./bin/download_wheels.sh
+
 8. **Upload to PyPI Test:** Upload the wheels to the PyPI test site using
    ``twine``.
 
@@ -147,6 +152,11 @@ This document describes the process for creating new releases.
 
    Do this at least for MacOS and Linux.
 
+   This process is automated. Run ./bin/pip_download_test.sh. 
+   This will download the ray from the test pypi repository and run the minimum 
+   sanity check from all the Python version supported. (3.5, 3.6, 3.7, 3.8)
+
+
 9. **Upload to PyPI:** Now that you've tested the wheels on the PyPI test
    repository, they can be uploaded to the main PyPI repository. Be careful,
    **it will not be possible to modify wheels once you upload them**, so any

doc/dev/bin/pip_download_test.sh

@@ -0,0 +1,60 @@
+# This script automatically download ray and run the sanity check (sanity_check.py)
+# in various Python version. This script requires conda command to exist.
+
+if [[ -z "$RAY_HASH" ]]; then
+    echo "RAY_HASH env var should be provided"
+    exit 1
+fi
+
+if [[ -z "$RAY_VERSION" ]]; then
+    echo "RAY_VERSION env var should be provided"
+    exit 1
+fi
+
+if ! [ -x "$(command -v conda)" ]; then
+    echo "conda doesn't exist. Please download conda for this machine"
+    exit 1
+else
+    echo "conda exists"
+fi
+
+echo "Start downloading Ray version ${RAY_VERSION} of commit ${RAY_HASH}"
+
+pip install --upgrade pip
+
+# This is required to use conda activate
+source "$(conda info --base)/etc/profile.d/conda.sh"
+
+for PYTHON_VERSION in "3.5" "3.6" "3.7" "3.8" 
+do
+    env_name="${RAY_VERSION}-${PYTHON_VERSION}-env"
+    conda create -y -n "${env_name}" python=${PYTHON_VERSION}
+    conda activate "${env_name}"
+    echo "\n\n\n========================================================="
+    echo "Python version."
+    python --version
+    echo "This should be equal to ${PYTHON_VERSION}"
+    echo "=========================================================\n\n\n"
+
+    pip install redis==3.3.2
+    pip install msgpack==0.6.2
+    pip install ray
+    pip uninstall -y ray
+    pip install --index-url https://test.pypi.org/simple/ ray
+
+    failed=false
+    echo "\n\n\n========================================================="
+    if python sanity_check.py; then
+        echo "PYTHON ${PYTHON_VERSION} succeed sanity check."
+    else
+        failed=true
+    fi
+    echo "=========================================================\n\n\n"
+
+    conda deactivate
+    conda remove -y --name ${env_name} --all
+    if [ "$failed" = true ]; then
+        echo "PYTHON ${PYTHON_VERSION} failed sanity check."
+        exit 1
+    fi
+done

doc/dev/bin/ray-project/cluster.yaml

@@ -0,0 +1,35 @@
+# This file is generated by `ray project create`.
+
+# A unique identifier for the head node and workers of this cluster.
+cluster_name: sanity-check
+
+# The maximum number of workers nodes to launch in addition to the head
+# node. This takes precedence over min_workers. min_workers defaults to 0.
+max_workers: 1
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-2
+    availability_zone: us-west-2a
+
+# How Ray will authenticate with newly launched nodes.
+auth:
+    ssh_user: ubuntu
+
+# Custom commands that will be run on the head node after common setup.
+head_setup_commands: 
+    # Install Anaconda.
+    - wget --quiet https://repo.continuum.io/archive/Anaconda3-5.0.1-Linux-x86_64.sh || true
+    - bash Anaconda3-5.0.1-Linux-x86_64.sh -b -p $HOME/anaconda3 || true
+    - echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc
+    - pip install -U pip 
+
+# Custom commands that will be run on worker nodes after common setup.
+worker_setup_commands: []
+
+# Command to start ray on the head node. You don't need to change this.
+head_start_ray_commands: []
+
+# Command to start ray on worker nodes. You don't need to change this.
+worker_start_ray_commands: []

doc/dev/bin/ray-project/project.yaml

@@ -0,0 +1,36 @@
+# This file is generated by `ray project create`.
+
+name: sanity-check
+
+# description: A short description of the project.
+# The URL of the repo this project is part of.
+# repo: ...
+
+cluster:
+  config: ray-project/cluster.yaml
+
+commands:
+  - name: sanity-check
+    help: "Do release sanity check with pip"
+    command: |      
+      export RAY_VERSION={{ray_version}}
+      export RAY_HASH={{commit}}
+      chmod +x ./pip_download_test.sh
+      ./pip_download_test.sh
+    params:
+    - name: ray_version # Ray version string.
+      default: "0.9.0.dev0"
+
+    - name: commit # Ray commit SHA string.
+      default: "FILL ME IN"
+
+# Pathnames for files and directories that should be saved
+# in a snapshot but that should not be synced with a# session. Pathnames can be relative to the project
+# directory or absolute. Generally, this should be files
+# that were created by an active session, such as
+# application checkpoints and logs.
+output_files: [
+  # For example, uncomment this to save the logs from the
+  # last ray job.
+  # "/tmp/ray/session_latest",
+]

doc/dev/bin/sanity_check.py

@@ -0,0 +1,31 @@
+import os
+import ray
+import sys
+
+RAY_VERSION = "RAY_VERSION"
+RAY_COMMIT = "RAY_HASH"
+
+ray_version = os.getenv(RAY_VERSION)
+ray_commit = os.getenv(RAY_COMMIT)
+
+if __name__ == "__main__":
+    print("Sanity check python version: {}".format(sys.version))
+    assert ray_version == ray.__version__, (
+        "Given Ray version {} is not matching with downloaded "
+        "version {}".format(ray_version, ray.__version__))
+    assert ray_commit == ray.__commit__, (
+        "Given Ray commit {} is not matching with downloaded "
+        "version {}".format(ray_commit, ray.__commit__))
+    assert ray.__file__ is not None
+
+    ray.init()
+    assert ray.is_initialized()
+
+    @ray.remote
+    def return_arg(arg):
+        return arg
+
+    val = 3
+    print("Running basic sanity check.")
+    assert ray.get(return_arg.remote(val)) == val
+    ray.shutdown()

doc/dev/download_wheels.sh

@@ -1,8 +1,21 @@
+if [[ -z "$RAY_HASH" ]]; then
+    echo "RAY_HASH env var should be provided"
+    exit 1
+fi
+
+if [[ -z "$RAY_VERSION" ]]; then
+    echo "RAY_VERSION env var should be provided"
+    exit 1
+fi
+
+# Make sure Linux wheels are downloadable without errors.
 wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp35-cp35m-manylinux1_x86_64.whl
 wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-manylinux1_x86_64.whl
 wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-manylinux1_x86_64.whl
 wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-manylinux1_x86_64.whl
+# Make sure macos wheels are downloadable without errors.
 wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp35-cp35m-macosx_10_13_intel.whl
 wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-macosx_10_13_intel.whl
 wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-macosx_10_13_intel.whl
+# Wheel name convention has been changed from Python 3.8.
 wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-macosx_10_13_x86_64.whl

doc/dev/get_contributors.py

@@ -3,6 +3,7 @@ from subprocess import check_output
 import shlex
 from tqdm import tqdm
 import click
+from collections import defaultdict
 
 
 @click.command()
@@ -63,6 +64,27 @@ def run(access_token, prev_release_commit, curr_release_commit):
     print()
     print("=" * 10)
 
+    # Organize commits
+    NO_CATEGORY = "[NO_CATEGORY]"
+
+    def get_category(line):
+        if line[0] == "[":
+            return (line.split("]")[0].strip(" ") + "]").upper()
+        else:
+            return NO_CATEGORY
+
+    commits = defaultdict(list)
+
+    with open("commits.txt") as file:
+        for line in file.readlines():
+            commits[get_category(line)].append(line.strip())
+
+    with open("commits.txt", "a") as file:
+        for category, commit_msgs in commits.items():
+            file.write("\n{}\n".format(category))
+            for commit_msg in commit_msgs:
+                file.write("{}\n".format(commit_msg))
+
 
 if __name__ == "__main__":
     run()

doc/dev/microbenchmark_analysis.py

@@ -0,0 +1,65 @@
+"""
+Compare the last two versions and output the change rate.
+This can also be used to draw graph, which is not
+implemented yet.
+
+Usage: python microbenchmark_analysis.py
+"""
+import glob
+
+from collections import defaultdict
+
+FRIST_VERSION = 0
+LAST_VERSION = 5
+FILES = sorted(
+    glob.glob("./release_logs/[0-9].[0-9].[0-9]/microbenchmark.txt"))
+
+task_info = defaultdict(list)
+task_std_info = defaultdict(list)
+version_list = []
+
+
+def get_task_type(line):
+    return line.split("per")[0]
+
+
+def get_task_performance(line):
+    return float(line.split(" ")[-3])
+
+
+def get_task_std(line):
+    return float(line.split(" ")[-1])
+
+
+def main():
+    for file_name in FILES:
+        version = file_name.split("/")[1]
+        version_list.append(version)
+
+        with open(file_name) as file:
+            for line in file.readlines():
+                if line.startswith("#") or line.startswith("\n"):
+                    continue
+                line = line.strip()
+                task_type = get_task_type(line)
+                task_performance = get_task_performance(line)
+                task_standard_deviation = get_task_std(line)
+
+                task_info[task_type].append(task_performance)
+                task_std_info[task_type].append(task_standard_deviation)
+
+    for task_type, task_performance_list in task_info.items():
+        # Newly introduced fields are not going to be compared.
+        if len(task_performance_list) < 2:
+            continue
+
+        latest_perf = task_performance_list[-1]
+        second_latest_perf = task_performance_list[-2]
+        change_rate = (
+            latest_perf - second_latest_perf) / second_latest_perf * 100
+        print("{} performance change rate: {}%".format(task_type,
+                                                       round(change_rate, 2)))
+
+
+if __name__ == "__main__":
+    main()

doc/dev/release_logs/0.8.5/microbenchmark.txt

@@ -0,0 +1,19 @@
+single client get calls per second 8214.4 +- 892.64
+single client put calls per second 5356.26 +- 36.53
+single client put gigabytes per second 13.46 +- 2.94
+multi client put calls per second 11351.56 +- 62.33
+multi client put gigabytes per second 16.44 +- 8.49
+single client tasks sync per second 1311.81 +- 66.92
+single client tasks async per second 15531.87 +- 220.75
+multi client tasks async per second 43140.09 +- 880.94
+1:1 actor calls sync per second 1916.18 +- 97.0
+1:1 actor calls async per second 5963.83 +- 115.07
+1:1 actor calls concurrent per second 5604.0 +- 222.31
+1:n actor calls async per second 10275.4 +- 167.94
+n:n actor calls async per second 34553.25 +- 196.74
+n:n actor calls with arg async per second 12522.36 +- 228.16
+1:1 async-actor calls sync per second 1096.88 +- 4.27
+1:1 async-actor calls async per second 3407.27 +- 74.07
+1:1 async-actor calls with args async per second 2406.66 +- 46.86
+1:n async-actor calls async per second 9109.89 +- 121.35
+n:n async-actor calls async per second 27338.51 +- 358.6

doc/dev/release_logs/0.8.5/rllib_regression.txt

@@ -0,0 +1,39 @@
+== Status ==
+Memory usage on this node: 15.5/480.3 GiB
+Using FIFO scheduling algorithm.
+Resources requested: 0/64 CPUs, 0.0/8 GPUs, 0.0/323.54 GiB heap, 0.0/98.39 GiB objects
+Result logdir: /home/ubuntu/ray_results/apex
+Result logdir: /home/ubuntu/ray_results/atari-a2c
+Result logdir: /home/ubuntu/ray_results/atari-basic-dqn
+Result logdir: /home/ubuntu/ray_results/atari-impala
+Result logdir: /home/ubuntu/ray_results/atari-ppo-tf
+Result logdir: /home/ubuntu/ray_results/atari-ppo-torch
+Number of trials: 24 (24 TERMINATED)
++-------------------------------------+------------+-------+--------+------------------+---------+----------+
+| Trial name                          | status     | loc   |   iter |   total time (s) |      ts |   reward |
+|-------------------------------------+------------+-------+--------+------------------+---------+----------|
+| IMPALA_BreakoutNoFrameskip-v4_00000 | TERMINATED |       |    300 |          3609.38 | 7596000 |   371.22 |
+| IMPALA_BreakoutNoFrameskip-v4_00001 | TERMINATED |       |    300 |          3604.49 | 7561500 |   361.02 |
+| IMPALA_BreakoutNoFrameskip-v4_00002 | TERMINATED |       |    300 |          3605.28 | 7581500 |   358.73 |
+| IMPALA_BreakoutNoFrameskip-v4_00003 | TERMINATED |       |    300 |          3604.35 | 7567500 |   336.82 |
+| PPO_BreakoutNoFrameskip-v4_00004    | TERMINATED |       |    682 |          3601.14 | 3410000 |    66.3  |
+| PPO_BreakoutNoFrameskip-v4_00005    | TERMINATED |       |    951 |          3602.81 | 4755000 |    36.08 |
+| PPO_BreakoutNoFrameskip-v4_00006    | TERMINATED |       |    951 |          3601.49 | 4755000 |    72.78 |
+| PPO_BreakoutNoFrameskip-v4_00007    | TERMINATED |       |    950 |          3601.34 | 4750000 |    38.93 |
+| PPO_BreakoutNoFrameskip-v4_00008    | TERMINATED |       |     27 |          3726.15 |  135000 |     5.77 |
+| PPO_BreakoutNoFrameskip-v4_00009    | TERMINATED |       |     27 |          3626.05 |  135000 |     2.16 |
+| PPO_BreakoutNoFrameskip-v4_00010    | TERMINATED |       |     22 |          3645.07 |  110000 |     2.84 |
+| PPO_BreakoutNoFrameskip-v4_00011    | TERMINATED |       |     22 |          3622.54 |  110000 |     2.32 |
+| APEX_BreakoutNoFrameskip-v4_00012   | TERMINATED |       |    106 |          3620.95 | 5372640 |    59.43 |
+| APEX_BreakoutNoFrameskip-v4_00013   | TERMINATED |       |    107 |          3617.3  | 8081280 |    97.22 |
+| APEX_BreakoutNoFrameskip-v4_00014   | TERMINATED |       |     88 |          3601.26 | 6635200 |    79.08 |
+| APEX_BreakoutNoFrameskip-v4_00015   | TERMINATED |       |     89 |          3618.23 | 6583040 |    85.28 |
+| A2C_BreakoutNoFrameskip-v4_00016    | TERMINATED |       |    353 |          3603.02 | 3351500 |   126.25 |
+| A2C_BreakoutNoFrameskip-v4_00017    | TERMINATED |       |    353 |          3607.96 | 3339000 |   125.67 |
+| A2C_BreakoutNoFrameskip-v4_00018    | TERMINATED |       |    351 |          3605.17 | 3306000 |   224.59 |
+| A2C_BreakoutNoFrameskip-v4_00019    | TERMINATED |       |    353 |          3607.46 | 3676000 |   163.83 |
+| DQN_BreakoutNoFrameskip-v4_00020    | TERMINATED |       |     27 |          3654.34 |  280104 |    15.51 |
+| DQN_BreakoutNoFrameskip-v4_00021    | TERMINATED |       |     27 |          3660.45 |  280104 |    17.15 |
+| DQN_BreakoutNoFrameskip-v4_00022    | TERMINATED |       |     27 |          3615    |  280104 |    15.37 |
+| DQN_BreakoutNoFrameskip-v4_00023    | TERMINATED |       |     28 |          3655.79 |  290108 |    16.02 |
++-------------------------------------+------------+-------+--------+------------------+---------+----------+

doc/dev/release_logs/0.8.5/stress_tests/application_stress_test.txt

@@ -0,0 +1,15 @@
+== Status ==
+Memory usage on this node: 23.4/480.3 GiB
+Using FIFO scheduling algorithm.
+Resources requested: 0/640 CPUs, 0/8 GPUs, 0.0/1905.86 GiB heap, 0.0/566.21 GiB objects
+Result logdir: /home/ubuntu/ray_results/atari-impala
+Number of trials: 4 (4 TERMINATED)
++------------------------------------------+------------+-------+-----------------------------+--------+------------------+----------+----------+
+| Trial name                               | status     | loc   | env                         |   iter |   total time (s) |       ts |   reward |
+|------------------------------------------+------------+-------+-----------------------------+--------+------------------+----------+----------|
+| IMPALA_BreakoutNoFrameskip-v4_00000      | TERMINATED |       | BreakoutNoFrameskip-v4      |    360 |          5974.47 | 30014500 |  488.4   |
+| IMPALA_BeamRiderNoFrameskip-v4_00001     | TERMINATED |       | BeamRiderNoFrameskip-v4     |    366 |          6026.01 | 30031500 |  424.84  |
+| IMPALA_QbertNoFrameskip-v4_00002         | TERMINATED |       | QbertNoFrameskip-v4         |    356 |          5897.19 | 30077500 | 5230.25  |
+| IMPALA_SpaceInvadersNoFrameskip-v4_00003 | TERMINATED |       | SpaceInvadersNoFrameskip-v4 |    361 |          5990.74 | 30103500 |  806.043 |
++------------------------------------------+------------+-------+-----------------------------+--------+------------------+----------+----------+
+

doc/dev/release_logs/0.8.5/stress_tests/test_dead_actors.txt

@@ -0,0 +1,4 @@
+Finished in: 98.98898577690125s
+Average iteration time: 0.9898880553245545s
+Max iteration time: 2.3596835136413574s
+Min iteration time: 0.1039724349975586s

doc/dev/release_logs/0.8.5/stress_tests/test_many_tasks.txt

@@ -0,0 +1,15 @@
+Stage 0 results:
+	Total time: 21.52851891517639
+Stage 1 results:
+	Total time: 151.462096452713
+	Average iteration time: 15.146199059486388
+	Max iteration time: 16.154610872268677
+	Min iteration time: 14.686655044555664
+Stage 2 results:
+	Total time: 1099.6661903858185
+	Average iteration time: 219.9327588558197
+	Max iteration time: 236.58895802497864
+	Min iteration time: 201.1623387336731
+Stage 3 results:
+	Actor creation time: 0.09940552711486816
+	Total time: 3180.518438100815