diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index bacb1ee55..6e572816a 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -19,12 +19,15 @@ jobs:
         include:
           - name: ubuntu
             os: ubuntu-16.04
+            python-version: 3.6
           - name: windows
             os: windows-2019
+            python-version: 3.8
           - name: macos
             os: macos-10.15
+            python-version: 3.6
     env:
-      PYTHON: 3.6
+      PYTHON: ${{ matrix.python-version }}
       TRAVIS_BUILD_DIR: ${{ github.workspace }}
       TRAVIS_COMMIT: ${{ github.sha }}
       #TRAVIS_PULL_REQUEST is defined in the scripts to account for "false"
diff --git a/build.sh b/build.sh
index 975a09b0b..23b373026 100755
--- a/build.sh
+++ b/build.sh
@@ -144,7 +144,7 @@ if [ "$RAY_BUILD_PYTHON" == "YES" ]; then
     TEMP_DIR="$(mktemp -d)"
     pushd "$TEMP_DIR"
     curl -f -s -L -R -o "pickle5-backport.zip" "https://github.com/pitrou/pickle5-backport/archive/c0c1a158f59366696161e0dffdd10cfe17601372.zip"
-    unzip pickle5-backport.zip
+    unzip -q pickle5-backport.zip
     pushd pickle5-backport-c0c1a158f59366696161e0dffdd10cfe17601372
       CC=gcc "$PYTHON_EXECUTABLE" setup.py --quiet bdist_wheel
       unzip -q -o dist/*.whl -d "$pickle5_path"
diff --git a/ci/travis/ci.sh b/ci/travis/ci.sh
index bbc3011ef..ab8340949 100755
--- a/ci/travis/ci.sh
+++ b/ci/travis/ci.sh
@@ -109,8 +109,13 @@ upload_wheels() {
 
 test_python() {
   if [ "${OSTYPE}" = msys ]; then
-    # Windows -- most tests won't work yet; just do the ones we know work
-    PYTHONPATH=python python -m pytest --durations=5 --timeout=300 python/ray/tests/test_mini.py
+    # Increased timeout from default of timeout=300 due to test_basic
+    bazel test -k --config=ci --test_timeout=600 --build_tests_only -- \
+      python/ray/tests:test_actor \
+      python/ray/tests:test_basic \
+      python/ray/tests:test_debug_tools \
+      python/ray/tests:test_mini \
+      ;
   fi
 }
 
@@ -190,7 +195,7 @@ install_ray() {
   (
     cd "${WORKSPACE_DIR}"/python
     build_dashboard_front_end
-    keep_alive pip install -e .
+    pip install -v -e .
   )
 }
 
diff --git a/ci/travis/install-bazel.sh b/ci/travis/install-bazel.sh
index 5a044a393..71a50f690 100755
--- a/ci/travis/install-bazel.sh
+++ b/ci/travis/install-bazel.sh
@@ -85,7 +85,6 @@ if [ "${TRAVIS-}" = true ] || [ -n "${GITHUB_WORKFLOW-}" ]; then
 build --color=yes
 build --curses=no
 build --disk_cache="$(test "${OSTYPE}" = msys || echo ~/ray-bazel-cache)"
-build --progress_report_interval=60
 # Use ray google cloud cache
 build --remote_cache="https://storage.googleapis.com/ray-bazel-cache"
 build --show_progress_rate_limit=15
diff --git a/python/ray/tests/BUILD b/python/ray/tests/BUILD
index 24cdbe789..234a7d71e 100644
--- a/python/ray/tests/BUILD
+++ b/python/ray/tests/BUILD
@@ -1,7 +1,15 @@
+SRCS = [] + select({
+    "@bazel_tools//src/conditions:windows": [
+        # TODO(mehrdadn): This should be added for all platforms once resulting errors are fixed
+        "conftest.py",
+    ],
+    "//conditions:default": [],
+})
+
 py_test(
     name = "test_actor",
     size = "medium",
-    srcs = ["test_actor.py"],
+    srcs = SRCS + ["test_actor.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -9,7 +17,7 @@ py_test(
 py_test(
     name = "test_actor_advanced",
     size = "medium",
-    srcs = ["test_actor_advanced.py"],
+    srcs = SRCS + ["test_actor_advanced.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -17,7 +25,7 @@ py_test(
 py_test(
     name = "test_actor_pool",
     size = "small",
-    srcs = ["test_actor_pool.py"],
+    srcs = SRCS + ["test_actor_pool.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -25,7 +33,7 @@ py_test(
 py_test(
     name = "test_memstat",
     size = "small",
-    srcs = ["test_memstat.py"],
+    srcs = SRCS + ["test_memstat.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -33,7 +41,7 @@ py_test(
 py_test(
     name = "test_iter",
     size = "medium",
-    srcs = ["test_iter.py"],
+    srcs = SRCS + ["test_iter.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -41,7 +49,7 @@ py_test(
 py_test(
     name = "test_actor_resources",
     size = "medium",
-    srcs = ["test_actor_resources.py"],
+    srcs = SRCS + ["test_actor_resources.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -49,7 +57,7 @@ py_test(
 py_test(
     name = "test_actor_failures",
     size = "medium",
-    srcs = ["test_actor_failures.py"],
+    srcs = SRCS + ["test_actor_failures.py"],
     # TODO(ekl) enable this once we support actor reconstruction again
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
@@ -58,7 +66,7 @@ py_test(
 py_test(
     name = "test_serialization",
     size = "small",
-    srcs = ["test_serialization.py"],
+    srcs = SRCS + ["test_serialization.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -66,7 +74,7 @@ py_test(
 py_test(
     name = "test_basic",
     size = "medium",
-    srcs = ["test_basic.py"],
+    srcs = SRCS + ["test_basic.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -74,7 +82,7 @@ py_test(
 py_test(
     name = "test_basic_2",
     size = "medium",
-    srcs = ["test_basic_2.py"],
+    srcs = SRCS + ["test_basic_2.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -82,7 +90,7 @@ py_test(
 py_test(
     name = "test_advanced",
     size = "medium",
-    srcs = ["test_advanced.py"],
+    srcs = SRCS + ["test_advanced.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -90,7 +98,7 @@ py_test(
 py_test(
     name = "test_advanced_2",
     size = "medium",
-    srcs = ["test_advanced_2.py"],
+    srcs = SRCS + ["test_advanced_2.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -98,7 +106,7 @@ py_test(
 py_test(
     name = "test_advanced_3",
     size = "medium",
-    srcs = ["test_advanced_3.py"],
+    srcs = SRCS + ["test_advanced_3.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -106,7 +114,7 @@ py_test(
 py_test(
     name = "test_component_failures",
     size = "small",
-    srcs = ["test_component_failures.py"],
+    srcs = SRCS + ["test_component_failures.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -114,7 +122,7 @@ py_test(
 py_test(
     name = "test_component_failures_2",
     size = "medium",
-    srcs = ["test_component_failures_2.py"],
+    srcs = SRCS + ["test_component_failures_2.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -122,7 +130,7 @@ py_test(
 py_test(
     name = "test_component_failures_3",
     size = "medium",
-    srcs = ["test_component_failures_3.py"],
+    srcs = SRCS + ["test_component_failures_3.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -130,7 +138,7 @@ py_test(
 py_test(
     name = "test_multinode_failures",
     size = "medium",
-    srcs = ["test_multinode_failures.py"],
+    srcs = SRCS + ["test_multinode_failures.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -138,7 +146,7 @@ py_test(
 py_test(
     name = "test_multinode_failures_2",
     size = "medium",
-    srcs = ["test_multinode_failures_2.py"],
+    srcs = SRCS + ["test_multinode_failures_2.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -146,7 +154,7 @@ py_test(
 py_test(
     name = "test_stress",
     size = "medium",
-    srcs = ["test_stress.py"],
+    srcs = SRCS + ["test_stress.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -154,7 +162,7 @@ py_test(
 py_test(
     name = "test_stress_sharded",
     size = "medium",
-    srcs = ["test_stress_sharded.py"],
+    srcs = SRCS + ["test_stress_sharded.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -162,7 +170,7 @@ py_test(
 py_test(
     name = "test_stress_failure",
     size = "large",
-    srcs = ["test_stress_failure.py"],
+    srcs = SRCS + ["test_stress_failure.py"],
     # TODO(ekl) enable again once we support direct call reconstruction
     tags = ["exclusive", "manual"],
     deps = ["//:ray_lib"],
@@ -171,42 +179,42 @@ py_test(
 py_test(
     name = "test_array",
     size = "medium",
-    srcs = ["test_array.py"],
+    srcs = SRCS + ["test_array.py"],
     deps = ["//:ray_lib"],
 )
 
 py_test(
     name = "test_autoscaler",
     size = "small",
-    srcs = ["test_autoscaler.py"],
+    srcs = SRCS + ["test_autoscaler.py"],
     deps = ["//:ray_lib"],
 )
 
 py_test(
     name = "test_autoscaler_yaml",
     size = "small",
-    srcs = ["test_autoscaler_yaml.py"],
+    srcs = SRCS + ["test_autoscaler_yaml.py"],
     deps = ["//:ray_lib"],
 )
 
 py_test(
     name = "test_cython",
     size = "small",
-    srcs = ["test_cython.py"],
+    srcs = SRCS + ["test_cython.py"],
     deps = ["//:ray_lib"],
 )
 
 py_test(
     name = "test_debug_tools",
     size = "small",
-    srcs = ["test_debug_tools.py"],
+    srcs = SRCS + ["test_debug_tools.py"],
     deps = ["//:ray_lib"],
 )
 
 py_test(
     name = "test_dynres",
     size = "medium",
-    srcs = ["test_dynres.py"],
+    srcs = SRCS + ["test_dynres.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -214,7 +222,7 @@ py_test(
 py_test(
     name = "test_failure",
     size = "medium",
-    srcs = ["test_failure.py"],
+    srcs = SRCS + ["test_failure.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -222,7 +230,7 @@ py_test(
 py_test(
     name = "test_reconstruction",
     size = "medium",
-    srcs = ["test_reconstruction.py"],
+    srcs = SRCS + ["test_reconstruction.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -230,7 +238,7 @@ py_test(
 py_test(
     name = "test_reference_counting",
     size = "medium",
-    srcs = ["test_reference_counting.py"],
+    srcs = SRCS + ["test_reference_counting.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -238,7 +246,7 @@ py_test(
 py_test(
     name = "test_reference_counting_2",
     size = "medium",
-    srcs = ["test_reference_counting_2.py"],
+    srcs = SRCS + ["test_reference_counting_2.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -246,7 +254,7 @@ py_test(
 py_test(
     name = "test_global_gc",
     size = "medium",
-    srcs = ["test_global_gc.py"],
+    srcs = SRCS + ["test_global_gc.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -254,7 +262,7 @@ py_test(
 py_test(
     name = "test_global_state",
     size = "small",
-    srcs = ["test_global_state.py"],
+    srcs = SRCS + ["test_global_state.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -262,7 +270,7 @@ py_test(
 py_test(
     name = "test_memory_limits",
     size = "medium",
-    srcs = ["test_memory_limits.py"],
+    srcs = SRCS + ["test_memory_limits.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -270,7 +278,7 @@ py_test(
 py_test(
     name = "test_memory_scheduling",
     size = "medium",
-    srcs = ["test_memory_scheduling.py"],
+    srcs = SRCS + ["test_memory_scheduling.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -278,7 +286,7 @@ py_test(
 py_test(
     name = "test_metrics",
     size = "small",
-    srcs = ["test_metrics.py"],
+    srcs = SRCS + ["test_metrics.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -286,7 +294,7 @@ py_test(
 py_test(
     name = "test_metrics_export",
     size = "small",
-    srcs = ["test_metrics_export.py"],
+    srcs = SRCS + ["test_metrics_export.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -294,7 +302,7 @@ py_test(
 py_test(
     name = "test_microbenchmarks",
     size = "small",
-    srcs = ["test_microbenchmarks.py"],
+    srcs = SRCS + ["test_microbenchmarks.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -302,14 +310,14 @@ py_test(
 py_test(
     name = "test_mini",
     size = "small",
-    srcs = ["test_mini.py"],
+    srcs = SRCS + ["test_mini.py"],
     deps = ["//:ray_lib"],
 )
 
 py_test(
     name = "test_multiprocessing",
     size = "medium",
-    srcs = ["test_multiprocessing.py"],
+    srcs = SRCS + ["test_multiprocessing.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -317,7 +325,7 @@ py_test(
 py_test(
     name = "test_joblib",
     size = "medium",
-    srcs = ["test_joblib.py"],
+    srcs = SRCS + ["test_joblib.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -325,7 +333,7 @@ py_test(
 py_test(
     name = "test_multi_node_2",
     size = "medium",
-    srcs = ["test_multi_node_2.py"],
+    srcs = SRCS + ["test_multi_node_2.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -333,7 +341,7 @@ py_test(
 py_test(
     name = "test_multi_node",
     size = "medium",
-    srcs = ["test_multi_node.py"],
+    srcs = SRCS + ["test_multi_node.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -341,14 +349,14 @@ py_test(
 py_test(
     name = "test_node_manager",
     size = "small",
-    srcs = ["test_node_manager.py"],
+    srcs = SRCS + ["test_node_manager.py"],
     deps = ["//:ray_lib"],
 )
 
 py_test(
     name = "test_object_manager",
     size = "medium",
-    srcs = ["test_object_manager.py"],
+    srcs = SRCS + ["test_object_manager.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -356,7 +364,7 @@ py_test(
 py_test(
     name = "test_projects",
     size = "small",
-    srcs = ["test_projects.py"],
+    srcs = SRCS + ["test_projects.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -364,21 +372,21 @@ py_test(
 py_test(
     name = "test_queue",
     size = "small",
-    srcs = ["test_queue.py"],
+    srcs = SRCS + ["test_queue.py"],
     deps = ["//:ray_lib"],
 )
 
 py_test(
     name = "test_ray_init",
     size = "small",
-    srcs = ["test_ray_init.py"],
+    srcs = SRCS + ["test_ray_init.py"],
     deps = ["//:ray_lib"],
 )
 
 py_test(
     name = "test_tempfile",
     size = "small",
-    srcs = ["test_tempfile.py"],
+    srcs = SRCS + ["test_tempfile.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -386,7 +394,7 @@ py_test(
 py_test(
     name = "test_tensorflow",
     size = "small",
-    srcs = ["test_tensorflow.py"],
+    srcs = SRCS + ["test_tensorflow.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -394,7 +402,7 @@ py_test(
 py_test(
     name = "test_unreconstructable_errors",
     size = "medium",
-    srcs = ["test_unreconstructable_errors.py"],
+    srcs = SRCS + ["test_unreconstructable_errors.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -402,7 +410,7 @@ py_test(
 py_test(
     name = "test_webui",
     size = "small",
-    srcs = ["test_webui.py"],
+    srcs = SRCS + ["test_webui.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -410,7 +418,7 @@ py_test(
 py_test(
     name = "test_args",
     size = "small",
-    srcs = ["test_args.py"],
+    srcs = SRCS + ["test_args.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -418,7 +426,7 @@ py_test(
 py_test(
     name = "test_asyncio",
     size = "small",
-    srcs = ["test_asyncio.py"],
+    srcs = SRCS + ["test_asyncio.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -426,7 +434,7 @@ py_test(
 py_test(
     name = "test_numba",
     size = "small",
-    srcs = ["test_numba.py"],
+    srcs = SRCS + ["test_numba.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
@@ -434,7 +442,7 @@ py_test(
 py_test(
     name = "test_cancel",
     size = "medium",
-    srcs = ["test_cancel.py"],
+    srcs = SRCS + ["test_cancel.py"],
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )