diff --git a/.buildkite/Dockerfile b/.buildkite/Dockerfile
index 86bd28148..d20a9170f 100644
--- a/.buildkite/Dockerfile
+++ b/.buildkite/Dockerfile
@@ -2,6 +2,8 @@ FROM ubuntu:focal
 
 ARG REMOTE_CACHE_URL
 ARG BUILDKITE_PULL_REQUEST
+ARG BUILDKITE_COMMIT
+ARG BUILDKITE_PULL_REQUEST_BASE_BRANCH
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV TZ=America/Los_Angeles
@@ -11,6 +13,9 @@ ENV CI=true
 ENV PYTHON=3.6
 ENV RAY_USE_RANDOM_PORTS=1
 ENV RAY_DEFAULT_BUILD=1
+ENV BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+ENV BUILDKITE_COMMIT=${BUILDKITE_COMMIT}
+ENV BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
 
 RUN apt-get update -qq
 RUN apt-get install -y -qq \
@@ -37,3 +42,7 @@ WORKDIR /ray
 COPY . .
 RUN ./ci/travis/ci.sh init
 RUN bash --login -i ./ci/travis/ci.sh build
+
+# Run determine test to run
+RUN bash --login -i -c "python ./ci/travis/determine_tests_to_run.py --output=json > affected_set.json"
+RUN cat affected_set.json
\ No newline at end of file
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index ebfd96322..00931f9dd 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,3 +1,30 @@
+- label: ":book: Lint"
+  commands:
+  - export LINT=1
+  - ./ci/travis/install-dependencies.sh
+  - ./ci/travis/ci.sh lint
+  - ./ci/travis/ci.sh build
+
+- label: ":java: Java"
+  commands:
+  - apt-get install -y openjdk-8-jdk maven clang-format
+  # Compile Java again so bazel will compile Java as a language.
+  - RAY_INSTALL_JAVA=1 ./ci/travis/ci.sh build
+  - ./java/test.sh
+
+- label: ":java: Streaming"
+  commands:
+  - apt-get install -y openjdk-8-jdk maven
+  # Compile Java again so bazel will compile Java as a language.
+  - RAY_INSTALL_JAVA=1 ./ci/travis/ci.sh build
+  - bazel test --config=ci $(./scripts/bazel_export_options)
+      //streaming:all
+  - bash streaming/src/test/run_streaming_queue_test.sh
+
+- label: ":cpp: Worker"
+  commands:
+  - ./ci/travis/ci.sh test_cpp
+
 - label: ":cpp: Tests"
   commands:
   - bazel test --config=ci $(./scripts/bazel_export_options)
diff --git a/ci/travis/build-docker-images.py b/ci/travis/build-docker-images.py
index ad69a15db..8283f5c8f 100644
--- a/ci/travis/build-docker-images.py
+++ b/ci/travis/build-docker-images.py
@@ -1,13 +1,12 @@
 import datetime
+import json
 import functools
 import glob
 import os
 import re
-import runpy
 import shutil
+import subprocess
 import sys
-from contextlib import redirect_stdout
-from io import StringIO
 from typing import List, Tuple
 
 import docker
@@ -69,18 +68,15 @@ def _get_wheel_name(minor_version_number):
 
 
 def _docker_affected():
-    result = StringIO()
-    with redirect_stdout(result):
-        runpy.run_path(
-            f"{_get_curr_dir()}/determine_tests_to_run.py",
-            run_name="__main__")
-    variable_definitions = result.getvalue().split()
-    env_var_dict = {
-        x.split("=")[0]: x.split("=")[1]
-        for x in variable_definitions
-    }
-    affected = env_var_dict["RAY_CI_DOCKER_AFFECTED"] == "1" or \
-        env_var_dict["RAY_CI_PYTHON_DEPENDENCIES_AFFECTED"] == "1"
+    proc = subprocess.run(
+        [
+            sys.executable, f"{_get_curr_dir()}/determine_tests_to_run.py",
+            "--output=json"
+        ],
+        capture_output=True)
+    affected_env_var_list = json.loads(proc.stdout)
+    affected = ("RAY_CI_DOCKER_AFFECTED" in affected_env_var_list or
+                "RAY_CI_PYTHON_DEPENDENCIES_AFFECTED" in affected_env_var_list)
     print(f"Docker affected: {affected}")
     return affected
 
diff --git a/ci/travis/ci.sh b/ci/travis/ci.sh
index 2527a4c5b..e72380bdb 100755
--- a/ci/travis/ci.sh
+++ b/ci/travis/ci.sh
@@ -357,9 +357,13 @@ lint_web() {
   (
     cd "${WORKSPACE_DIR}"/python/ray/new_dashboard/client
     set +x # suppress set -x since it'll get very noisy here
-    . "${HOME}/.nvm/nvm.sh"
+
+    if [ -z "${BUILDKITE-}" ]; then
+      . "${HOME}/.nvm/nvm.sh"
+      nvm use --silent node
+    fi
+
     install_npm_project
-    nvm use --silent node
     local filenames
     # shellcheck disable=SC2207
     filenames=($(find src -name "*.ts" -or -name "*.tsx"))
diff --git a/ci/travis/determine_tests_to_run.py b/ci/travis/determine_tests_to_run.py
index cba016fcf..be37a2946 100644
--- a/ci/travis/determine_tests_to_run.py
+++ b/ci/travis/determine_tests_to_run.py
@@ -9,6 +9,7 @@ import re
 import subprocess
 import sys
 from pprint import pformat
+import argparse
 
 
 def list_changed_files(commit_range):
@@ -30,7 +31,44 @@ def list_changed_files(commit_range):
     return [s.strip() for s in out.decode().splitlines() if s is not None]
 
 
+def is_pull_request():
+    event_type = None
+
+    for key in ["GITHUB_EVENT_NAME", "TRAVIS_EVENT_TYPE"]:
+        event_type = os.getenv(key, event_type)
+
+    if (os.environ.get("BUILDKITE")
+            and os.environ.get("BUILDKITE_PULL_REQUEST") != "false"):
+        event_type = "pull_request"
+
+    return event_type == "pull_request"
+
+
+def get_commit_range():
+    commit_range = None
+
+    if os.environ.get("TRAVIS"):
+        commit_range = os.environ["TRAVIS_COMMIT_RANGE"]
+    elif os.environ.get("GITHUB_EVENT_PATH"):
+        with open(os.environ["GITHUB_EVENT_PATH"], "rb") as f:
+            event = json.loads(f.read())
+        base = event["pull_request"]["base"]["sha"]
+        commit_range = "{}...{}".format(base, event.get("after", ""))
+    elif os.environ.get("BUILDKITE"):
+        commit_range = "{}...{}".format(
+            os.environ["BUILDKITE_PULL_REQUEST_BASE_BRANCH"],
+            os.environ["BUILDKITE_COMMIT"],
+        )
+
+    assert commit_range is not None
+    return commit_range
+
+
 if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--output", type=str, help="json or envvars", default="envvars")
+    args = parser.parse_args()
 
     RAY_CI_TUNE_AFFECTED = 0
     RAY_CI_SGD_AFFECTED = 0
@@ -50,20 +88,10 @@ if __name__ == "__main__":
     RAY_CI_DOC_AFFECTED = 0
     RAY_CI_PYTHON_DEPENDENCIES_AFFECTED = 0
 
-    event_type = None
-    for key in ["GITHUB_EVENT_NAME", "TRAVIS_EVENT_TYPE"]:
-        event_type = os.getenv(key, event_type)
-
-    if event_type == "pull_request":
-
-        commit_range = os.getenv("TRAVIS_COMMIT_RANGE")
-        if commit_range is None:
-            with open(os.environ["GITHUB_EVENT_PATH"], "rb") as f:
-                event = json.loads(f.read())
-            base = event["pull_request"]["base"]["sha"]
-            commit_range = "{}...{}".format(base, event.get("after", ""))
+    if is_pull_request():
+        commit_range = get_commit_range()
         files = list_changed_files(commit_range)
-
+        print(pformat(commit_range), file=sys.stderr)
         print(pformat(files), file=sys.stderr)
 
         skip_prefix_list = [
@@ -187,7 +215,7 @@ if __name__ == "__main__":
         RAY_CI_ONLY_RLLIB_AFFECTED = 1
 
     # Log the modified environment variables visible in console.
-    print(" ".join([
+    output_string = " ".join([
         "RAY_CI_TUNE_AFFECTED={}".format(RAY_CI_TUNE_AFFECTED),
         "RAY_CI_SGD_AFFECTED={}".format(RAY_CI_SGD_AFFECTED),
         "RAY_CI_ONLY_RLLIB_AFFECTED={}".format(RAY_CI_ONLY_RLLIB_AFFECTED),
@@ -209,4 +237,15 @@ if __name__ == "__main__":
         "RAY_CI_DOCKER_AFFECTED={}".format(RAY_CI_DOCKER_AFFECTED),
         "RAY_CI_PYTHON_DEPENDENCIES_AFFECTED={}".format(
             RAY_CI_PYTHON_DEPENDENCIES_AFFECTED),
-    ]))
+    ])
+
+    # Debug purpose
+    print(output_string, file=sys.stderr)
+
+    # Used by buildkite log format
+    if args.output.lower() == "json":
+        pairs = [item.split("=") for item in output_string.split(" ")]
+        affected_vars = [key for key, affected in pairs if affected == "1"]
+        print(json.dumps(affected_vars))
+    else:
+        print(output_string)
diff --git a/java/test.sh b/java/test.sh
index 86afc719b..a842194e6 100755
--- a/java/test.sh
+++ b/java/test.sh
@@ -16,6 +16,16 @@ pushd "$ROOT_DIR"
   mvn -T16 checkstyle:check
 popd
 
+on_exit() {
+  exit_code=$?
+  if [ $exit_code -ne 0 ]; then
+    echo "Exit trap, printing ray logs"
+    cat /tmp/ray/session_latest/logs/*
+  fi
+}
+
+trap on_exit EXIT
+
 run_testng() {
     local exit_code
     if "$@"; then