diff --git a/ci/asan_tests/run.sh b/ci/asan_tests/run.sh index 9dde1e53f..fbc2a1571 100644 --- a/ci/asan_tests/run.sh +++ b/ci/asan_tests/run.sh @@ -8,7 +8,7 @@ usage() { for i in "$@" do -case $i in +case "$i" in --git-sha=*) git_sha="${i#*=}" ;; @@ -26,8 +26,8 @@ done echo "git-sha: $git_sha" ./run_asan_tests.sh setup -if [[ git_sha != "" ]] +if [ -n "${git_sha}" ] then - git_sha=$git_sha ./run_asan_tests.sh recompile + git_sha="${git_sha}" ./run_asan_tests.sh recompile fi ./run_asan_tests.sh run diff --git a/ci/long_running_distributed_tests/run.sh b/ci/long_running_distributed_tests/run.sh index 8e52eaedc..d9edef56f 100644 --- a/ci/long_running_distributed_tests/run.sh +++ b/ci/long_running_distributed_tests/run.sh @@ -11,8 +11,8 @@ usage() { for i in "$@" do -echo $i -case $i in +echo "$i" +case "$i" in --ray-version=*) ray_version="${i#*=}" diff --git a/ci/long_running_tests/run.sh b/ci/long_running_tests/run.sh index 32a81c729..171256fa4 100644 --- a/ci/long_running_tests/run.sh +++ b/ci/long_running_tests/run.sh @@ -11,8 +11,8 @@ usage() { for i in "$@" do -echo $i -case $i in +echo "$i" +case "$i" in --ray-version=*) ray_version="${i#*=}" @@ -51,7 +51,7 @@ echo "workload: $workload" wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl" pip install -U pip -source activate tensorflow_p36 && pip install -q -U $wheel Click -source activate tensorflow_p36 && pip install -q ray[all] gym[atari] +source activate tensorflow_p36 && pip install -q -U "$wheel" Click +source activate tensorflow_p36 && pip install -q "ray[all]" "gym[atari]" source activate tensorflow_p36 && python "workloads/$workload.py" diff --git a/ci/microbenchmark/run.sh b/ci/microbenchmark/run.sh index 1c66e9ce8..a1fcc0b6a 100644 --- a/ci/microbenchmark/run.sh +++ b/ci/microbenchmark/run.sh @@ -10,7 +10,7 @@ usage() { for i in "$@" do -case $i in +case "$i" in --ray-version=*) ray_version="${i#*=}" @@ -32,7 +32,7 @@ case $i in esac done -if [[ $ray_version == "" || $commit == "" || $ray_branch == "" ]] +if [ -z "$ray_version" ] || [ -z "$commit" ] || [ -z "$ray_branch" ] then echo "Provide --ray-version, --commit, and --ray-branch" exit 1 @@ -42,10 +42,10 @@ echo "version: $ray_version" echo "commit: $commit" echo "branch: $ray_branch" -rm ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl || true -wget https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl +rm "ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl" || true +wget "https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl" pip uninstall -y -q ray -pip install -U ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl +pip install -U "ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl" OMP_NUM_THREADS=64 ray microbenchmark diff --git a/ci/regression_test/stress_tests/run.sh b/ci/regression_test/stress_tests/run.sh index 22f2ef9e4..e56b2b8e5 100644 --- a/ci/regression_test/stress_tests/run.sh +++ b/ci/regression_test/stress_tests/run.sh @@ -11,8 +11,8 @@ usage() { for i in "$@" do -echo $i -case $i in +echo "$i" +case "$i" in --ray-version=*) ray_version="${i#*=}" diff --git a/ci/travis/bazel-preclean.sh b/ci/travis/bazel-preclean.sh deleted file mode 100755 index cf42a580b..000000000 --- a/ci/travis/bazel-preclean.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# Args: [Bazel-target] -# -# This script cleans up any genrule() outputs in the transitive dependencies of the provided target. -# -# This is useful for forcing genrule actions to re-run, because the _true_ outputs of those actions -# can include a larger set of files (e.g. files copied to the workspace) which Bazel is unable to -# detect changes to (or delete changes of). -# -# Usually, you would run this script along with 'git clean -f', to make sure Bazel re-copies outputs -# the next time a build occurs. - -( - set -euo pipefail - bazel aquery --output=textproto \ - "mnemonic(\"Genrule\", deps(${1-//:*}))" | awk ' - { - body = 0; - } - /^^ / { - body = 1; - } - /^^\S.* {$/ { - section = $1; - delete arr; - } - body { - if (section == "artifacts") { - p = $2; - if ($1 == "exec_path:") { - p = substr(p, 2, length(p) - 2); # strip quotes - } - arr[$1] = p; - } - } - /^^}/ { - artifacts[arr["id:"]] = arr["exec_path:"]; # save the ID -> artifact mapping - } - /^^ *output_ids:/ { - print(artifacts[$2]); # print the output artifact - } - ' | tr "\n" "\0" | xargs -0 -r -- rm -f -- -) diff --git a/ci/travis/bazel.py b/ci/travis/bazel.py new file mode 100755 index 000000000..3b01c1d7c --- /dev/null +++ b/ci/travis/bazel.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python + +import ast +import errno +import json +import os +import re +import subprocess +import stat +import sys + +from collections import defaultdict, OrderedDict + + +def textproto_format(space, key, value, json_encoder): + """Rewrites a key-value pair from textproto as JSON.""" + if value.startswith(b"\""): + evaluated = ast.literal_eval(value.decode("utf-8")) + value = json_encoder.encode(evaluated).encode("utf-8") + return b"%s[\"%s\", %s]" % (space, key, value) + + +def textproto_split(input_lines, json_encoder): + """When given e.g. the output of "bazel aquery --output=textproto", + yields each top-level item as a string formatted as JSON (if an encoder is + given) or Python AST. + The input MUST be formatted neatly line-by-line, as follows: + actions { + mnemonic: "Genrule" + environment_variables { + key: "CC" + value: "clang" + } + ... + } + targets { + id: "0" + label: "//:target" + rule_class_id: "0" + } + """ + outputs = [] + re_flags = re.M + pat_open = re.compile(b"^(\\s*)([-\\w:]+)(\\s*){$", flags=re_flags) + pat_line = re.compile(b"^(\\s*)([-\\w]+): (.*)$", flags=re_flags) + pat_close = re.compile(b"}$", flags=re_flags) + prev_comma = False + prev_tail = b"" + for full_line in input_lines: + pieces = re.split(b"(\\r|\\n)", full_line, 1) + pieces[1:] = [b"".join(pieces[1:])] + [line, tail] = pieces + next_line = pat_open.sub(b"\\1[\"\\2\",\\3[", line) + outputs.append(b"" if not prev_comma else b"]" + if next_line.endswith(b"}") else b",") + next_line = pat_close.sub(b"]", next_line) + next_line = pat_line.sub( + lambda m: textproto_format(*(m.groups() + (json_encoder, ))), + next_line) + outputs.append(prev_tail + next_line) + if line == b"}": + yield b"".join(outputs) + del outputs[:] + prev_comma = line != b"}" and (next_line.endswith(b"]") + or next_line.endswith(b"\"")) + prev_tail = tail + if len(outputs) > 0: + yield b"".join(outputs) + del outputs[:] + + +def textproto_parse(stream, encoding, json_encoder): + for item in textproto_split(stream, json_encoder): + yield json.loads(item) + + +class Bazel(object): + encoding = "utf-8" + + def __init__(self, program=None): + if program is None: + program = os.getenv("BAZEL_EXECUTABLE", "bazel") + self.argv = (program, ) + self.extra_args = ("--show_progress=no", ) + + def _call(self, command, *args): + return subprocess.check_output( + self.argv + (command, ) + args[:1] + self.extra_args + args[1:], + stdin=subprocess.PIPE) + + def info(self, *args): + result = OrderedDict() + for line in self._call("info", *args).splitlines(): + (key, value) = line.split(b":", 1) + if value.startswith(b" "): + value = value[1:] + result[key.decode(self.encoding)] = value.decode(self.encoding) + return result + + def aquery(self, *args): + lines = self._call("aquery", "--output=textproto", *args).splitlines() + return textproto_parse(lines, self.encoding, json.JSONEncoder()) + + +def parse_aquery_shell_calls(aquery_results): + """Extracts and yields the command lines representing the genrule() rules + from Bazel aquery results. + """ + for (key, val) in aquery_results: + if key == "actions": + [mnemonic] = [pair[1] for pair in val if pair[0] == "mnemonic"] + if mnemonic == "Genrule": + yield [pair[1] for pair in val if pair[0] == "arguments"] + + +def parse_aquery_output_artifacts(aquery_results): + """Extracts and yields the file paths representing the output artifact + from the provided Bazel aquery results. + """ + artifacts = {} + for (key, val) in aquery_results: + if key == "artifacts": + [artifact_id] = [pair[1] for pair in val if pair[0] == "id"] + [exec_path] = [pair[1] for pair in val if pair[0] == "exec_path"] + artifacts[artifact_id] = exec_path + elif key == "actions": + output_ids = [pair[1] for pair in val if pair[0] == "output_ids"] + for output_id in output_ids: + yield artifacts[output_id] + + +def textproto2json(infile, outfile): + """Translates the output of bazel aquery --output=textproto into JSON. + Useful for later command-line manipulation. + + Args: + infile: The binary input stream. + outfile: The binary output stream. + """ + json_encoder = json.JSONEncoder(indent=2) + encoding = "utf-8" + for obj in textproto_parse(infile, encoding, json_encoder): + outfile.write((json_encoder.encode(obj) + "\n").encode(encoding)) + + +def preclean(bazel_aquery): + """Cleans up any genrule() outputs for the provided target(s). + + This is useful for forcing genrule actions to re-run, because the _true_ + outputs of those actions can include a larger set of files (e.g. files + copied to the workspace) which Bazel is unable to detect changes to (or + delete changes of). + + Usually, you would run this script along with 'git clean -f', to make sure + Bazel re-copies outputs the next time a build occurs. + """ + result = 0 + bazel = Bazel() + aquery_results = bazel.aquery("--include_artifacts=true", bazel_aquery) + for path in parse_aquery_output_artifacts(aquery_results): + try: + if sys.platform == "win32": + os.chmod(path, stat.S_IWRITE) # Needed to remove read-only bit + os.remove(path) + except IOError as ex: + if ex.errno != errno.ENOENT: + sys.stderr.write(str(ex) + "\n") + result = result or ex.errno + return result + + +def shellcheck(bazel_aquery, *shellcheck_argv): + """Runs shellcheck with the provided argument(s) on all targets that match + the given Bazel aquery. + + Args: + bazel_aquery: A Bazel aquery expression (e.g. "//:*") + shellcheck_argv: The command-line arguments to call for shellcheck. + Note that the first entry should be the shellcheck program itself. + If omitted, will simply call "shellcheck". + + Returns: + The exit code of shellcheck. + """ + bazel = Bazel() + shellcheck_argv = list(shellcheck_argv) or ["shellcheck"] + all_script_infos = defaultdict(lambda: []) + aquery_results = bazel.aquery("--include_artifacts=false", bazel_aquery) + shell_calls = list(parse_aquery_shell_calls(aquery_results)) + for shell_args in shell_calls: + shname = os.path.basename(os.path.splitext(shell_args[0])[0]).lower() + finished_options = False + i = 1 + while i < len(shell_args): + if finished_options or not shell_args[i].startswith("-"): + all_script_infos[shname].append((shell_args[i], None)) + elif shell_args[i] == "--": + finished_options = True + elif shell_args[i] in ("-o", "+o"): + i += 1 + elif shell_args[i] == "-c": + all_script_infos[shname].append((None, shell_args[i + 1])) + break + i += 1 + + result = 0 + bazel_execution_root = None + for shell, script_infos in all_script_infos.items(): + scripts_combined = [] + has_stdin = False + filenames = [] + for script_file, script_text in script_infos: + if script_file is not None: + filenames.append(script_file) + if script_text is not None: + has_stdin = True + flatc = "host/bin/external/com_github_google_flatbuffers/flatc" + if flatc not in script_text: + statements = ["if test -t 0; then", script_text, "fi"] + scripts_combined.append("\n".join(statements)) + if has_stdin: + filenames.insert(0, "-") + if shell.endswith("sh"): + if bazel_execution_root is None: + bazel_execution_root = bazel.info()["execution_root"] + cwd = bazel_execution_root + cmdargs = ["--shell=" + shell, "--external-sources"] + filenames + cmdargs = shellcheck_argv + cmdargs + proc = subprocess.Popen(cmdargs, stdin=subprocess.PIPE, cwd=cwd) + try: + proc.communicate("\n".join(scripts_combined).encode("utf-8")) + finally: + proc.wait() + result = result or proc.returncode + return result + + +def main(program, command, *command_args): + result = 0 + if command == textproto2json.__name__: + result = textproto2json(sys.stdin.buffer, sys.stdout.buffer, + *command_args) + elif command == shellcheck.__name__: + result = shellcheck(*command_args) + elif command == preclean.__name__: + result = preclean(*command_args) + else: + raise ValueError("Unrecognized command: " + command) + return result + + +if __name__ == "__main__": + sys.exit(main(*sys.argv) or 0) diff --git a/ci/travis/check_import_order.py b/ci/travis/check_import_order.py index 71b5c2adf..bf349ca89 100644 --- a/ci/travis/check_import_order.py +++ b/ci/travis/check_import_order.py @@ -9,6 +9,7 @@ some/file/path.py:23 import psutil without explicitly import ray before it. """ import argparse +import glob import io import re import sys @@ -64,7 +65,7 @@ if __name__ == "__main__": file_path = Path(args.path) if file_path.is_dir(): - all_py_files = file_path.rglob("*.py") + all_py_files = glob.glob("*.py", recursive=True) else: all_py_files = [file_path] diff --git a/ci/travis/ci.sh b/ci/travis/ci.sh index a2bf2d8c0..8fcfadaed 100755 --- a/ci/travis/ci.sh +++ b/ci/travis/ci.sh @@ -310,12 +310,7 @@ lint_readme() { fi } -lint_python() { - # ignore dict vs {} (C408), others are defaults - command -V python - python -m flake8 --inline-quotes '"' --no-avoid-escape \ - --exclude=python/ray/core/generated/,streaming/python/generated,doc/source/conf.py,python/ray/cloudpickle/,python/ray/thirdparty_files \ - --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605 +lint_scripts() { "${ROOT_DIR}"/format.sh --all } @@ -361,8 +356,8 @@ _lint() { { echo "WARNING: Skipping linting C/C++ as clang-format is not installed."; } 2> /dev/null fi - # Run Python linting - lint_python + # Run script linting + lint_scripts # Make sure that the README is formatted properly. lint_readme diff --git a/ci/travis/format.sh b/ci/travis/format.sh index d3ad4004d..ca0530a9c 100755 --- a/ci/travis/format.sh +++ b/ci/travis/format.sh @@ -3,10 +3,11 @@ # You are encouraged to run this locally before pushing changes for review. # Cause the script to exit if a single command fails -set -eo pipefail +set -euo pipefail FLAKE8_VERSION_REQUIRED="3.7.7" YAPF_VERSION_REQUIRED="0.23.0" +SHELLCHECK_VERSION_REQUIRED="0.7.1" check_command_exist() { VERSION="" @@ -17,6 +18,9 @@ check_command_exist() { flake8) VERSION=$FLAKE8_VERSION_REQUIRED ;; + shellcheck) + VERSION=$SHELLCHECK_VERSION_REQUIRED + ;; *) echo "$1 is not a required dependency" exit 1 @@ -49,6 +53,7 @@ fi FLAKE8_VERSION=$(flake8 --version | awk '{print $1}') YAPF_VERSION=$(yapf --version | awk '{print $2}') +SHELLCHECK_VERSION=$(shellcheck --version | awk '/^version:/ {print $2}') # params: tool name, tool version, required version tool_version_check() { @@ -59,6 +64,7 @@ tool_version_check() { tool_version_check "flake8" "$FLAKE8_VERSION" "$FLAKE8_VERSION_REQUIRED" tool_version_check "yapf" "$YAPF_VERSION" "$YAPF_VERSION_REQUIRED" +tool_version_check "shellcheck" "$SHELLCHECK_VERSION" "$SHELLCHECK_VERSION_REQUIRED" if which clang-format >/dev/null; then CLANG_FORMAT_VERSION=$(clang-format --version | awk '{print $3}') @@ -70,6 +76,16 @@ fi # Only fetch master since that's the branch we're diffing against. git fetch upstream master || true +SHELLCHECK_FLAGS=( + --exclude=1090 # "Can't follow non-constant source. Use a directive to specify location." + --exclude=1091 # "Not following {file} due to some error" + --exclude=2207 # "Prefer mapfile or read -a to split command output (or quote to avoid splitting)." -- these aren't compatible with macOS's old Bash +) + +SHELLCHECK_BAZEL_FLAGS=( + --exclude=2043 # "This loop will only ever run once. Bad quoting or missing glob/expansion?" -- Bazel preprocessing can trigger this needlessly +) + YAPF_FLAGS=( '--style' "$ROOT/.style.yapf" '--recursive' @@ -83,9 +99,65 @@ YAPF_EXCLUDES=( '--exclude' 'python/ray/thirdparty_files/*' ) +shellcheck_scripts() { + shellcheck "${SHELLCHECK_FLAGS[@]}" "$@" +} + +shellcheck_bazel() { + "${ROOT}"/ci/travis/bazel.py shellcheck "mnemonic(\"Genrule\", deps(//:*))" shellcheck "${SHELLCHECK_FLAGS[@]}" "${SHELLCHECK_BAZEL_FLAGS[@]}" "$@" +} + # Format specified files format() { - yapf --in-place "${YAPF_FLAGS[@]}" -- "$@" + local shell_files=() python_files=() bazel_files=() + + local name + for name in "$@"; do + local base="${name%.*}" + local suffix="${name#${base}}" + + local shebang="" + read -r shebang < "${name}" || true + case "${shebang}" in + '#!'*) + shebang="${shebang#/usr/bin/env }" + shebang="${shebang%% *}" + shebang="${shebang##*/}" + ;; + esac + + if [ "${base}" = "WORKSPACE" ] || [ "${base}" = "BUILD" ] || [ "${suffix}" = ".BUILD" ] || [ "${suffix}" = ".bazel" ] || [ "${suffix}" = ".bzl" ]; then + bazel_files+=("${name}") + elif [ -z "${suffix}" ] && [ "${shebang}" != "${shebang#python}" ] || [ "${suffix}" != "${suffix#.py}" ]; then + python_files+=("${name}") + elif [ -z "${suffix}" ] && [ "${shebang}" != "${shebang%sh}" ] || [ "${suffix}" != "${suffix#.sh}" ]; then + shell_files+=("${name}") + else + echo "error: failed to determine file type: ${name}" 1>&2 + return 1 + fi + done + + if [ 0 -lt "${#python_files[@]}" ]; then + yapf --in-place "${YAPF_FLAGS[@]}" -- "${python_files[@]}" + fi + + if shellcheck --shell=sh --format=diff - < /dev/null; then + if [ 0 -lt "${#bazel_files[@]}" ]; then + if ! shellcheck_bazel; then + echo "Bazel genrule() scripts cannot be fixed automatically; please fix manually." 1>&2 + shellcheck_bazel --format=diff + fi + fi + if [ 0 -lt "${#shell_files[@]}" ]; then + local difference + difference="$(shellcheck_scripts --format=diff "${shell_files[@]}" || true && printf "-")" + difference="${difference%-}" + printf "%s" "${difference}" | patch -p1 + fi + else + echo "error: this version of shellcheck does not support diffs" + fi } # Format files that differ from main branch. Ignores dirs that are not slated @@ -121,20 +193,49 @@ format_changed() { clang-format -i fi fi + + if command -v shellcheck >/dev/null; then + if ! git diff --diff-filter=ACRM --quiet --exit-code "$MERGEBASE" -- 'WORKSPACE' 'WORKSPACE.*' 'BUILD.*' '*.bzl' '*.bazel' &>/dev/null; then + shellcheck_bazel + fi + + local shell_files + # shellcheck disable=SC2207 + shell_files=($( + git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.sh' && + git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- ':(exclude)*.*' | xargs -r git --no-pager grep -l '^#!\(/usr\)\?/bin/\(env \+\)\?\(ba\)\?sh' + )) + if [ 0 -lt "${#shell_files[@]}" ]; then + shellcheck_scripts "${shell_files[@]}" + fi + fi } # Format all files, and print the diff to stdout for travis. format_all() { + flake8 --inline-quotes '"' --no-avoid-escape --exclude=python/ray/core/generated/,streaming/python/generated,doc/source/conf.py,python/ray/cloudpickle/,python/ray/thirdparty_files/ --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605 + yapf --diff "${YAPF_FLAGS[@]}" "${YAPF_EXCLUDES[@]}" test python + + local shell_files + # shellcheck disable=SC2207 + shell_files=($( + git -C "${ROOT}" ls-files --exclude-standard HEAD -- "*.sh" && + git -C "${ROOT}" --no-pager grep -l '^#!\(/usr\)\?/bin/\(env \+\)\?\(ba\)\?sh' ":(exclude)*.sh" + )) + if [ 0 -lt "${#shell_files[@]}" ]; then + shellcheck_scripts "${shell_files[@]}" + fi + shellcheck_bazel } # This flag formats individual files. --files *must* be the first command line # arg to use this option. -if [[ "$1" == '--files' ]]; then +if [ "${1-}" == '--files' ]; then format "${@:2}" # If `--all` is passed, then any further arguments are ignored and the # entire python directory is formatted. -elif [[ "$1" == '--all' ]]; then +elif [ "${1-}" == '--all' ]; then format_all else # Format only the files that changed in last commit. diff --git a/ci/travis/install-dependencies.sh b/ci/travis/install-dependencies.sh index 46d9c2bae..8504ee804 100755 --- a/ci/travis/install-dependencies.sh +++ b/ci/travis/install-dependencies.sh @@ -136,8 +136,34 @@ install_miniconda() { test -x "${CONDA_PYTHON_EXE}" # make sure conda is activated } +install_shellcheck() { + local shellcheck_version="0.7.1" + if [ "${shellcheck_version}" != "$(command -v shellcheck > /dev/null && shellcheck --version | sed -n "s/version: //p")" ]; then + local osname="" + case "${OSTYPE}" in + linux*) osname="linux";; + darwin*) osname="darwin";; + esac + local name="shellcheck-v${shellcheck_version}" + if [ "${osname}" = linux ] || [ "${osname}" = darwin ]; then + sudo mkdir -p /usr/local/bin || true + curl -f -s -L "https://github.com/koalaman/shellcheck/releases/download/v${shellcheck_version}/${name}.${osname}.x86_64.tar.xz" | { + sudo tar -C /usr/local/bin -x -v -J --strip-components=1 "${name}/shellcheck" + } + else + mkdir -p /usr/local/bin + curl -f -s -L -o "${name}.zip" "https://github.com/koalaman/shellcheck/releases/download/v${shellcheck_version}/${name}.zip" + unzip "${name}.zip" "${name}.exe" + mv -f "${name}.exe" "/usr/local/bin/shellcheck.exe" + fi + test "${shellcheck_version}" = "$(shellcheck --version | sed -n "s/version: //p")" + fi +} + install_linters() { pip install -r "${WORKSPACE_DIR}"/python/requirements_linters.txt + + install_shellcheck } install_nvm() { diff --git a/doc/dev/download_wheels.sh b/doc/dev/download_wheels.sh index 08cefd351..2baf276a7 100644 --- a/doc/dev/download_wheels.sh +++ b/doc/dev/download_wheels.sh @@ -22,6 +22,6 @@ wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_H # Wheel name convention has been changed from Python 3.8. wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-macosx_10_13_x86_64.whl" # Make sure Windows wheels are downloadable without errors. -wget https://ray-wheels.s3-us-west-2.amazonaws.com/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-win_amd64.whl -wget https://ray-wheels.s3-us-west-2.amazonaws.com/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-win_amd64.whl -wget https://ray-wheels.s3-us-west-2.amazonaws.com/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-win_amd64.whl +wget "https://ray-wheels.s3-us-west-2.amazonaws.com/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-win_amd64.whl" +wget "https://ray-wheels.s3-us-west-2.amazonaws.com/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-win_amd64.whl" +wget "https://ray-wheels.s3-us-west-2.amazonaws.com/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-win_amd64.whl" diff --git a/python/build-wheel-windows.sh b/python/build-wheel-windows.sh index 2b24de5d3..a1c0be136 100755 --- a/python/build-wheel-windows.sh +++ b/python/build-wheel-windows.sh @@ -8,6 +8,10 @@ WORKSPACE_DIR="${ROOT_DIR}/.." PY_VERSIONS=($(python -s -c "import runpy, sys; runpy.run_path(sys.argv.pop(), run_name='__api__')" python_versions "${ROOT_DIR}"/setup.py | tr -d "\r")) PY_SCRIPT_SUBDIR=Scripts # 'bin' for UNIX, 'Scripts' for Windows +bazel_preclean() { + "${WORKSPACE_DIR}"/ci/travis/bazel.py preclean "mnemonic(\"Genrule\", deps(//:*))" +} + get_python_version() { python -s -c "import sys; sys.stdout.write('%s.%s' % sys.version_info[:2])" } @@ -56,7 +60,7 @@ build_wheel_windows() { local local_dir="python/dist" for pyversion in "${pyversions[@]}"; do if [ -z "${pyversion}" ]; then continue; fi - "${WORKSPACE_DIR}"/ci/travis/bazel-preclean.sh + bazel_preclean git clean -q -f -f -x -d -e "${local_dir}" -e python/ray/dashboard/client git checkout -q -f -- . @@ -82,7 +86,7 @@ build_wheel_windows() { ) done - "${WORKSPACE_DIR}"/ci/travis/bazel-preclean.sh + bazel_preclean if [ 0 -eq "${ray_uninstall_status}" ]; then # If Ray was previously installed, restore it install_ray fi