From f3c1248d98d321dc774ab2bf388df90e532df078 Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Sun, 10 Sep 2017 13:41:16 -0700 Subject: [PATCH] Clone catapult and generate html files during installation. (#956) * Clone catapult and generate static html during setup. * Include UI files in installation. * Fix directory to clone catapult to and fix linting. * Use absolute path. * Make sure we find a sufficiently new version of python2 when building wheels. * Copy the trace_viewer_full.html file to the local directory if it is not present. * Make sure wheels fail to build if UI is not included. --- python/build-wheel-macos.sh | 2 +- python/build-wheel-manylinux1.sh | 8 +++- python/ray/experimental/ui.py | 74 +++++------------------------ python/setup.py | 2 + src/thirdparty/build_flatbuffers.sh | 0 src/thirdparty/build_thirdparty.sh | 3 ++ src/thirdparty/build_ui.sh | 43 +++++++++++++++++ 7 files changed, 67 insertions(+), 65 deletions(-) mode change 100644 => 100755 src/thirdparty/build_flatbuffers.sh create mode 100755 src/thirdparty/build_ui.sh diff --git a/python/build-wheel-macos.sh b/python/build-wheel-macos.sh index cce1e509f..a20ee5ecd 100755 --- a/python/build-wheel-macos.sh +++ b/python/build-wheel-macos.sh @@ -58,7 +58,7 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do $PIP_CMD install wheel # Add the correct Python to the path and build the wheel. This is only # needed so that the installation finds the cython executable. - PATH=$MACPYTHON_PY_PREFIX/$PY_MM/bin:$PATH $PYTHON_EXE setup.py bdist_wheel + INCLUDE_UI=1 PATH=$MACPYTHON_PY_PREFIX/$PY_MM/bin:$PATH $PYTHON_EXE setup.py bdist_wheel mv dist/*.whl ../.whl/ popd done diff --git a/python/build-wheel-manylinux1.sh b/python/build-wheel-manylinux1.sh index e79b8ef4e..14e94e219 100755 --- a/python/build-wheel-manylinux1.sh +++ b/python/build-wheel-manylinux1.sh @@ -6,6 +6,12 @@ echo 10 EOF chmod +x /usr/bin/nproc +# Remove this old Python 2.4.3 executable, and make the "python2" command find +# a newer version of Python. We need this for autogenerating some files for the +# UI. +rm -f /usr/bin/python2 +ln -s /opt/python/cp27-cp27m/bin/python2 /usr/bin/python2 + mkdir .whl for PYTHON in cp27-cp27mu cp33-cp33m cp34-cp34m cp35-cp35m cp36-cp36m; do # The -f flag is passed twice to also run git clean in the arrow subdirectory. @@ -16,7 +22,7 @@ for PYTHON in cp27-cp27mu cp33-cp33m cp34-cp34m cp35-cp35m cp36-cp36m; do # Fix the numpy version because this will be the oldest numpy version we can # support. /opt/python/${PYTHON}/bin/pip install numpy==1.10.4 cython - PATH=/opt/python/${PYTHON}/bin:$PATH /opt/python/${PYTHON}/bin/python setup.py bdist_wheel + INCLUDE_UI=1 PATH=/opt/python/${PYTHON}/bin:$PATH /opt/python/${PYTHON}/bin/python setup.py bdist_wheel # In the future, run auditwheel here. mv dist/*.whl ../.whl/ popd diff --git a/python/ray/experimental/ui.py b/python/ray/experimental/ui.py index d0a2b3702..d1da9702c 100644 --- a/python/ray/experimental/ui.py +++ b/python/ray/experimental/ui.py @@ -4,7 +4,6 @@ import os import pprint import ray import shutil -import subprocess import tempfile import time @@ -289,65 +288,6 @@ def _get_temp_file_path(**kwargs): return os.path.relpath(temp_file_path) -# Helper function that ensures that catapult is cloned to the correct location -# and that the HTML files required for task trace embedding are in the same -# directory as the web UI. -def _setup_trace_dependencies(): - catapult_home = "/tmp/ray/catapult" - catapult_commit = "33a9271eb3cf5caf925293ec6a4b47c94f1ac968" - try: - # Check if we're inside a git repo - cmd = ["git", - "-C", - catapult_home, - "rev-parse", - "--is-inside-work-tree"] - subprocess.check_call(cmd) - - except subprocess.CalledProcessError: - # Error on non-zero exit code (e.g. - ".git not found") - if not os.path.exists(os.path.join(catapult_home)): - print( - "Cloning catapult to {} (this may take a while...)".format( - catapult_home)) - cmd = ["git", - "clone", - "https://github.com/catapult-project/catapult.git", - catapult_home] - subprocess.check_call(cmd) - - # Checks out the commit associated with allowing different arrow - # colors. This can and should be removed after catapult's next - # release. - print("Checking out commit {}.".format(catapult_commit)) - cmd = ["git", "-C", catapult_home, "checkout", catapult_commit] - subprocess.check_call(cmd) - - # Path to the embedded trace viewer HTML file. - embedded_trace_path = os.path.join(catapult_home, - "tracing", - "bin", - "index.html") - # Checks that the trace viewer renderer file exists, generates it if it - # doesn't. - if not os.path.exists("trace_viewer_full.html"): - vulcanize_bin = os.path.join(catapult_home, - "tracing", - "bin", - "vulcanize_trace_viewer") - # TODO(rkn): The vulcanize_trace_viewer script currently requires - # Python 2. Remove this dependency. - cmd = ["python2", - vulcanize_bin, - "--config", - "chrome", - "--output", - "trace_viewer_full.html"] - subprocess.check_call(cmd) - - return catapult_home, embedded_trace_path - - def task_timeline(): path_input = widgets.Button(description="View task timeline") @@ -381,6 +321,14 @@ def task_timeline(): display(widgets.HBox([label_options, breakdown_opt])) display(path_input) + # Check that the trace viewer renderer file is present, and copy it to the + # current working directory if it is not present. + if not os.path.exists("trace_viewer_full.html"): + shutil.copy( + os.path.join(os.path.dirname(os.path.abspath(__file__)), + "../core/src/catapult_files/trace_viewer_full.html"), + "trace_viewer_full.html") + def handle_submit(sender): json_tmp = tempfile.mktemp() + ".json" @@ -425,9 +373,9 @@ def task_timeline(): print("Opening html file in browser...") - # Check that the catapult repo is cloned to the correct location - print(_setup_trace_dependencies()) - catapult_home, trace_viewer_path = _setup_trace_dependencies() + trace_viewer_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "../core/src/catapult_files/index.html") html_file_path = _get_temp_file_path(suffix=".html") json_file_path = _get_temp_file_path(suffix=".json") diff --git a/python/setup.py b/python/setup.py index 2ff65d6a1..2684b55be 100644 --- a/python/setup.py +++ b/python/setup.py @@ -23,6 +23,8 @@ ray_files = [ "ray/core/src/local_scheduler/local_scheduler", "ray/core/src/local_scheduler/liblocal_scheduler_library.so", "ray/core/src/global_scheduler/global_scheduler", + "ray/core/src/catapult_files/index.html", + "ray/core/src/catapult_files/trace_viewer_full.html", "ray/WebUI.ipynb" ] diff --git a/src/thirdparty/build_flatbuffers.sh b/src/thirdparty/build_flatbuffers.sh old mode 100644 new mode 100755 diff --git a/src/thirdparty/build_thirdparty.sh b/src/thirdparty/build_thirdparty.sh index 702825516..d0c6e808a 100755 --- a/src/thirdparty/build_thirdparty.sh +++ b/src/thirdparty/build_thirdparty.sh @@ -42,6 +42,9 @@ else FLATBUFFERS_HOME="" fi +# Clone catapult and build the static HTML needed for the UI. +bash "$TP_DIR/build_ui.sh" + echo "building arrow" cd $TP_DIR/arrow/cpp mkdir -p $TP_DIR/arrow/cpp/build diff --git a/src/thirdparty/build_ui.sh b/src/thirdparty/build_ui.sh new file mode 100755 index 000000000..17d4372d7 --- /dev/null +++ b/src/thirdparty/build_ui.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +set -x + +# Cause the script to exit if a single command fails. +set -e + +TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +CATAPULT_COMMIT=33a9271eb3cf5caf925293ec6a4b47c94f1ac968 +CATAPULT_HOME=$TP_DIR/catapult +VULCANIZE_BIN=$CATAPULT_HOME/tracing/bin/vulcanize_trace_viewer + +CATAPULT_FILES=$TP_DIR/../../python/ray/core/src/catapult_files + +# This is where we will copy the files that need to be packaged with the wheels. +mkdir -p $CATAPULT_FILES + +if [ ! type python2 > /dev/null ]; then + echo "cannot properly set up UI without a python2 executable" + if [ "$INCLUDE_UI" == "1" ]; then + # Since the UI is explicitly supposed to be included, fail here. + exit 1 + else + # Let installation continue without building the UI. + exit 0 + fi +fi + +# Download catapult and use it to autogenerate some static html if it isn't +# already present. +if [ ! -d $CATAPULT_HOME ]; then + echo "setting up catapult" + # Clone the catapult repository. + git clone https://github.com/catapult-project/catapult.git $CATAPULT_HOME + # Check out the appropriate commit from catapult. + pushd $CATAPULT_HOME + git checkout $CATAPULT_COMMIT + popd + + python2 $VULCANIZE_BIN --config chrome --output $CATAPULT_FILES/trace_viewer_full.html + cp $CATAPULT_HOME/tracing/bin/index.html $CATAPULT_FILES/index.html +fi