diff --git a/BUILD.bazel b/BUILD.bazel
index 1f70aa8a9..ea62c4d8a 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -19,6 +19,11 @@ config_setting(
     flag_values = {"@bazel_tools//tools/cpp:compiler": "clang-cl"},
 )
 
+config_setting(
+    name = "opt",
+    values = {"compilation_mode": "opt"},
+)
+
 # === Begin of protobuf definitions ===
 
 proto_library(
diff --git a/bazel/ray.bzl b/bazel/ray.bzl
index 1b79e29b9..38133d16a 100644
--- a/bazel/ray.bzl
+++ b/bazel/ray.bzl
@@ -3,6 +3,9 @@ load("@com_github_checkstyle_java//checkstyle:checkstyle.bzl", "checkstyle_test"
 load("@bazel_common//tools/maven:pom_file.bzl", "pom_file")
 
 COPTS = ["-DRAY_USE_GLOG"] + select({
+    "//:opt": ["-DBAZEL_OPT"],
+    "//conditions:default": [],
+}) + select({
     "@bazel_tools//src/conditions:windows": [
         # TODO(mehrdadn): (How to) support dynamic linking?
         "-DRAY_STATIC",
diff --git a/ci/travis/ci.sh b/ci/travis/ci.sh
index d36e9efe5..a2bf2d8c0 100755
--- a/ci/travis/ci.sh
+++ b/ci/travis/ci.sh
@@ -133,9 +133,11 @@ test_core() {
 }
 
 test_python() {
+  local pathsep=":" args=()
   if [ "${OSTYPE}" = msys ]; then
-    local args=(python/ray/tests/...)
+    pathsep=";"
     args+=(
+      python/ray/tests/...
       -python/ray/tests:test_advanced_2
       -python/ray/tests:test_advanced_3  # test_invalid_unicode_in_worker_log() fails on Windows
       -python/ray/tests:test_autoscaler_aws
@@ -157,7 +159,16 @@ test_python() {
       -python/ray/tests:test_stress_sharded  # timeout
       -python/ray/tests:test_webui
     )
-    bazel test -k --config=ci --test_timeout=600 --build_tests_only -- "${args[@]}";
+  fi
+  if [ 0 -lt "${#args[@]}" ]; then  # Any targets to test?
+    install_ray
+    # TODO(mehrdadn): We set PYTHONPATH here to let Python find our pickle5 under pip install -e.
+    # It's unclear to me if this should be necessary, but this is to make tests run for now.
+    # Check why this issue doesn't arise on Linux/Mac.
+    # Ideally importing ray.cloudpickle should import pickle5 automatically.
+    bazel test -k --config=ci --test_timeout=600 --build_tests_only \
+      --test_env=PYTHONPATH="${PYTHONPATH-}${pathsep}${WORKSPACE_DIR}/python/ray/pickle5_files" -- \
+      "${args[@]}";
   fi
 }
 
@@ -283,7 +294,7 @@ build_wheels() {
       suppress_output "${WORKSPACE_DIR}"/python/build-wheel-macos.sh
       ;;
     msys*)
-      suppress_output "${WORKSPACE_DIR}"/python/build-wheel-windows.sh
+      keep_alive "${WORKSPACE_DIR}"/python/build-wheel-windows.sh
       ;;
   esac
 }
diff --git a/python/build-wheel-windows.sh b/python/build-wheel-windows.sh
index b9b6e14a3..2b24de5d3 100755
--- a/python/build-wheel-windows.sh
+++ b/python/build-wheel-windows.sh
@@ -5,7 +5,7 @@ set -euxo pipefail
 ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)"
 WORKSPACE_DIR="${ROOT_DIR}/.."
 
-PY_VERSIONS=(3.6 3.7 3.8)
+PY_VERSIONS=($(python -s -c "import runpy, sys; runpy.run_path(sys.argv.pop(), run_name='__api__')" python_versions "${ROOT_DIR}"/setup.py | tr -d "\r"))
 PY_SCRIPT_SUBDIR=Scripts  # 'bin' for UNIX, 'Scripts' for Windows
 
 get_python_version() {
@@ -39,7 +39,7 @@ install_ray() {
 uninstall_ray() {
   pip uninstall -y ray
 
-  rm -r -f "${WORKSPACE_DIR}"/python/ray/thirdparty_files
+  python -s -c "import runpy, sys; runpy.run_path(sys.argv.pop(), run_name='__api__')" clean "${ROOT_DIR}"/setup.py
 }
 
 build_wheel_windows() {
diff --git a/python/ray/_raylet.pxd b/python/ray/_raylet.pxd
index 2b8a573a6..4c8ae1c3f 100644
--- a/python/ray/_raylet.pxd
+++ b/python/ray/_raylet.pxd
@@ -32,6 +32,9 @@ cdef extern from *:
     #undef __OPTIMIZE__
     int __OPTIMIZE__ = 1;
     #define __OPTIMIZE__ 1
+    #elif defined(BAZEL_OPT)
+    // For compilers that don't define __OPTIMIZE__
+    int __OPTIMIZE__ = 1;
     #else
     int __OPTIMIZE__ = 0;
     #endif
diff --git a/python/setup.py b/python/setup.py
index e417d96ba..f4f6c4798 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -1,4 +1,5 @@
 import argparse
+import errno
 import glob
 import io
 import logging
@@ -31,6 +32,11 @@ SUPPORTED_BAZEL = (3, 2, 0)
 ROOT_DIR = os.path.dirname(__file__)
 BUILD_JAVA = os.getenv("RAY_INSTALL_JAVA") == "1"
 
+PICKLE5_SUBDIR = os.path.join("ray", "pickle5_files")
+THIRDPARTY_SUBDIR = os.path.join("ray", "thirdparty_files")
+
+CLEANABLE_SUBDIRS = [PICKLE5_SUBDIR, THIRDPARTY_SUBDIR]
+
 exe_suffix = ".exe" if sys.platform == "win32" else ""
 
 # .pyd is the extension Python requires on Windows for shared libraries.
@@ -232,7 +238,7 @@ def build(build_python, build_java):
         except ImportError:
             pass
     if not pickle5:
-        download_pickle5(os.path.join(ROOT_DIR, "ray", "pickle5_files"))
+        download_pickle5(os.path.join(ROOT_DIR, PICKLE5_SUBDIR))
 
     # Note: We are passing in sys.executable so that we use the same
     # version of Python to build packages inside the build.sh script. Note
@@ -243,7 +249,7 @@ def build(build_python, build_java):
         subprocess.check_call(
             [
                 sys.executable, "-m", "pip", "install", "-q",
-                "--target=" + os.path.join(ROOT_DIR, "ray", "thirdparty_files")
+                "--target=" + os.path.join(ROOT_DIR, THIRDPARTY_SUBDIR)
             ] + pip_packages,
             env=dict(os.environ, CC="gcc"))
 
@@ -329,10 +335,10 @@ def pip_run(build_ext):
 
     # We also need to install pickle5 along with Ray, so make sure that the
     # relevant non-Python pickle5 files get copied.
-    pickle5_dir = os.path.join(ROOT_DIR, "ray", "pickle5_files")
+    pickle5_dir = os.path.join(ROOT_DIR, PICKLE5_SUBDIR)
     files_to_include += walk_directory(os.path.join(pickle5_dir, "pickle5"))
 
-    thirdparty_dir = os.path.join(ROOT_DIR, "ray", "thirdparty_files")
+    thirdparty_dir = os.path.join(ROOT_DIR, THIRDPARTY_SUBDIR)
     files_to_include += walk_directory(thirdparty_dir)
 
     # Copy over the autogenerated protobuf Python bindings.
@@ -355,7 +361,7 @@ def pip_run(build_ext):
 
 def api_main(program, *args):
     parser = argparse.ArgumentParser()
-    choices = ["build", "bazel_version", "help"]
+    choices = ["build", "bazel_version", "python_versions", "clean", "help"]
     parser.add_argument("command", type=str, choices=choices)
     parser.add_argument(
         "-l",
@@ -381,6 +387,22 @@ def api_main(program, *args):
         result = build(**kwargs)
     elif parsed_args.command == "bazel_version":
         print(".".join(map(str, SUPPORTED_BAZEL)))
+    elif parsed_args.command == "python_versions":
+        for version in SUPPORTED_PYTHONS:
+            # NOTE: On Windows this will print "\r\n" on the command line.
+            # Strip it out by piping to tr -d "\r".
+            print(".".join(map(str, version)))
+    elif parsed_args.command == "clean":
+
+        def onerror(function, path, excinfo):
+            nonlocal result
+            if excinfo[1].errno != errno.ENOENT:
+                msg = excinfo[1].strerror
+                logger.error("cannot remove {}: {}" % (path, msg))
+                result = 1
+
+        for subdir in CLEANABLE_SUBDIRS:
+            shutil.rmtree(os.path.join(ROOT_DIR, subdir), onerror=onerror)
     elif parsed_args.command == "help":
         parser.print_help()
     else: