Fix release directory & RELEASE_PROCESS.md (#12269)

release/util/download_wheels.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+if [[ -z "$RAY_HASH" ]]; then
+    echo "RAY_HASH env var should be provided"
+    exit 1
+fi
+
+if [[ -z "$RAY_VERSION" ]]; then
+    echo "RAY_VERSION env var should be provided"
+    exit 1
+fi
+
+# Linux.
+wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-manylinux2014_x86_64.whl"
+wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-manylinux2014_x86_64.whl"
+wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-manylinux2014_x86_64.whl"
+
+# macOS.
+wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-macosx_10_13_intel.whl"
+wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-macosx_10_13_intel.whl"
+wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-macosx_10_13_x86_64.whl"
+
+# Windows.
+wget "https://ray-wheels.s3-us-west-2.amazonaws.com/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-win_amd64.whl"
+wget "https://ray-wheels.s3-us-west-2.amazonaws.com/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-win_amd64.whl"
+wget "https://ray-wheels.s3-us-west-2.amazonaws.com/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-win_amd64.whl"

release/util/get_contributors.py

@@ -0,0 +1,90 @@
+from github import Github
+from subprocess import check_output
+import shlex
+from tqdm import tqdm
+import click
+from collections import defaultdict
+
+
+@click.command()
+@click.option(
+    "--access-token",
+    required=True,
+    help="""
+Github Access token that has repo:public_repo and user:read:user permission.
+
+Create them at https://github.com/settings/tokens/new
+""",
+)
+@click.option(
+    "--prev-release-commit",
+    required=True,
+    help="Last commit SHA of the previous release.")
+@click.option(
+    "--curr-release-commit",
+    required=True,
+    help="Last commit SHA of the current release.")
+def run(access_token, prev_release_commit, curr_release_commit):
+    print("Writing commit descriptions to 'commits.txt'...")
+    check_output(
+        (f"git log {prev_release_commit}..{curr_release_commit} "
+         f"--pretty=format:'%s' > commits.txt"),
+        shell=True)
+    # Generate command
+    cmd = []
+    cmd.append((f"git log {prev_release_commit}..{curr_release_commit} "
+                f"--pretty=format:\"%s\" "
+                f" | grep -Eo \"#(\d+)\""))
+    joined = " && ".join(cmd)
+    cmd = f"bash -c '{joined}'"
+    cmd = shlex.split(cmd)
+    print("Executing", cmd)
+
+    # Sort the PR numbers
+    pr_numbers = [
+        int(l.lstrip("#")) for l in check_output(cmd).decode().split()
+    ]
+    print("PR numbers", pr_numbers)
+
+    # Use Github API to fetch the
+    g = Github(access_token)
+    ray_repo = g.get_repo("ray-project/ray")
+    logins = set()
+    for num in tqdm(pr_numbers):
+        try:
+            logins.add(ray_repo.get_pull(num).user.login)
+        except Exception as e:
+            print(e)
+
+    print()
+    print("Here's the list of contributors")
+    print("=" * 10)
+    print()
+    print("@" + ", @".join(logins))
+    print()
+    print("=" * 10)
+
+    # Organize commits
+    NO_CATEGORY = "[NO_CATEGORY]"
+
+    def get_category(line):
+        if line[0] == "[":
+            return (line.split("]")[0].strip(" ") + "]").upper()
+        else:
+            return NO_CATEGORY
+
+    commits = defaultdict(list)
+
+    with open("commits.txt") as file:
+        for line in file.readlines():
+            commits[get_category(line)].append(line.strip())
+
+    with open("commits.txt", "a") as file:
+        for category, commit_msgs in commits.items():
+            file.write("\n{}\n".format(category))
+            for commit_msg in commit_msgs:
+                file.write("{}\n".format(commit_msg))
+
+
+if __name__ == "__main__":
+    run()

release/util/kickoff.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+
+source "$2"
+
+ray_version=${ray_version:-}
+commit=${commit:-}
+
+if [[ $ray_version == "" || $commit == "" || $1 == "" ]]
+then
+    echo "Provide --ray-version, --commit, and --ray-branch"
+    exit 1
+fi
+
+echo "version: $ray_version"
+echo "commit: $commit"
+echo "workload: $1"
+
+DATESTR=$(date +%Y%m%d-%H%M)
+SESSION="$1-$DATESTR"
+
+echo "session: $SESSION"
+
+chmod +x ./run.sh
+if [ -z "$NO_UP" ]; then
+  anyscale up "$SESSION"
+fi
+anyscale push "$SESSION"
+anyscale exec -n "$SESSION" "./run.sh $1 --ray-version=$ray_version --commit=$commit"

release/util/microbenchmark_analysis.py

@@ -0,0 +1,64 @@
+"""
+Compare the last two versions and output the change rate.
+This can also be used to draw graph, which is not
+implemented yet.
+
+Usage: python microbenchmark_analysis.py
+"""
+import glob
+
+from collections import defaultdict
+
+FRIST_VERSION = 0
+LAST_VERSION = 5
+FILES = sorted(
+    glob.glob("./release_logs/[0-9].[0-9].[0-9]/microbenchmark.txt"))
+
+task_info = defaultdict(list)
+task_std_info = defaultdict(list)
+version_list = []
+
+
+def get_task_type(line):
+    return line.split("per")[0]
+
+
+def get_task_performance(line):
+    return float(line.split(" ")[-3])
+
+
+def get_task_std(line):
+    return float(line.split(" ")[-1])
+
+
+def main():
+    for file_name in FILES:
+        version = file_name.split("/")[1]
+        version_list.append(version)
+
+        with open(file_name) as file:
+            for line in file.readlines():
+                if line.startswith("#") or line.startswith("\n"):
+                    continue
+                line = line.strip()
+                task_type = get_task_type(line)
+                task_performance = get_task_performance(line)
+                task_standard_deviation = get_task_std(line)
+                task_info[task_type].append(task_performance)
+                task_std_info[task_type].append(task_standard_deviation)
+
+    for task_type, task_performance_list in task_info.items():
+        # Newly introduced fields are not going to be compared.
+        if len(task_performance_list) < 2:
+            continue
+
+        latest_perf = task_performance_list[-1]
+        second_latest_perf = task_performance_list[-2]
+        change_rate = (
+            latest_perf - second_latest_perf) / second_latest_perf * 100
+        print("{} performance change rate: {}%".format(task_type,
+                                                       round(change_rate, 2)))
+
+
+if __name__ == "__main__":
+    main()

release/util/pip_download_test.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+# This script automatically download ray and run the sanity check (sanity_check.py)
+# in various Python version. This script requires conda command to exist.
+
+unset RAY_ADDRESS
+export RAY_HASH=$RAY_HASH
+export RAY_VERSION=$RAY_VERSION
+
+if [[ -z "$RAY_HASH" ]]; then
+    echo "RAY_HASH env var should be provided"
+    exit 1
+fi
+
+if [[ -z "$RAY_VERSION" ]]; then
+    echo "RAY_VERSION env var should be provided"
+    exit 1
+fi
+
+if ! [ -x "$(command -v conda)" ]; then
+    echo "conda doesn't exist. Please download conda for this machine"
+    exit 1
+else
+    echo "conda exists"
+fi
+
+echo "Start downloading Ray version ${RAY_VERSION} of commit ${RAY_HASH}"
+
+pip install --upgrade pip
+
+# This is required to use conda activate
+source "$(conda info --base)/etc/profile.d/conda.sh"
+
+for PYTHON_VERSION in "3.6" "3.7" "3.8" 
+do
+    env_name="${RAY_VERSION}-${PYTHON_VERSION}-env"
+    conda create -y -n "${env_name}" python=${PYTHON_VERSION}
+    conda activate "${env_name}"
+    printf "\n\n\n"
+    echo "========================================================="
+    echo "Python version."
+    python --version
+    echo "This should be equal to ${PYTHON_VERSION}"
+    echo "========================================================="
+    printf "\n\n\n"
+
+    pip install redis==3.3.2
+    pip install msgpack==1.0.0
+    pip install aioredis
+    pip install colorful
+    pip install prometheus-client==0.7.1
+    pip install opencensus
+    pip install gpustat
+    pip install ray
+    pip uninstall -y ray
+    pip install --index-url https://test.pypi.org/simple/ ray
+
+    failed=false
+    printf "\n\n\n"
+    echo "========================================================="
+    if python sanity_check.py; then
+        echo "PYTHON ${PYTHON_VERSION} succeed sanity check."
+    else
+        failed=true
+    fi
+    echo "========================================================="
+    printf "\n\n\n"
+
+    conda deactivate
+    conda remove -y --name "${env_name}" --all
+    if [ "$failed" = true ]; then
+        echo "PYTHON ${PYTHON_VERSION} failed sanity check."
+        exit 1
+    fi
+done

release/util/sanity_check.py

@@ -0,0 +1,31 @@
+import os
+import ray
+import sys
+
+RAY_VERSION = "RAY_VERSION"
+RAY_COMMIT = "RAY_HASH"
+
+ray_version = os.getenv(RAY_VERSION)
+ray_commit = os.getenv(RAY_COMMIT)
+
+if __name__ == "__main__":
+    print("Sanity check python version: {}".format(sys.version))
+    assert ray_version == ray.__version__, (
+        "Given Ray version {} is not matching with downloaded "
+        "version {}".format(ray_version, ray.__version__))
+    assert ray_commit == ray.__commit__, (
+        "Given Ray commit {} is not matching with downloaded "
+        "version {}".format(ray_commit, ray.__commit__))
+    assert ray.__file__ is not None
+
+    ray.init()
+    assert ray.is_initialized()
+
+    @ray.remote
+    def return_arg(arg):
+        return arg
+
+    val = 3
+    print("Running basic sanity check.")
+    assert ray.get(return_arg.remote(val)) == val
+    ray.shutdown()