diff --git a/python/README-benchmarks.rst b/python/README-benchmarks.rst
index c992151b5..aa1acca38 100644
--- a/python/README-benchmarks.rst
+++ b/python/README-benchmarks.rst
@@ -3,5 +3,8 @@ Running the benchmarks
 
 You can run the benchmark suite by doing the following:
 
-1. Install https://github.com/airspeed-velocity/asv: ``pip install asv``
+1. Install https://github.com/ray-project/asv: ``cd asv; pip install -e .``
 2. Run ``asv dev`` in this directory.
+
+To run ASV inside docker, you can use the following command:
+`docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA bash -c '/ray/test/jenkins_tests/run_asv.sh'`
diff --git a/python/benchmarks/benchmarks.py b/python/benchmarks/benchmarks.py
index b5b6202ad..32adccc13 100644
--- a/python/benchmarks/benchmarks.py
+++ b/python/benchmarks/benchmarks.py
@@ -7,7 +7,9 @@ import ray.test.test_functions as test_functions
 
 
 def setup():
-    ray.init(num_workers=4, num_cpus=4)
+    if not hasattr(setup, "is_initialized"):
+        ray.init(num_workers=4, num_cpus=4)
+        setup.is_initialized = True
 
 
 class TimeSuite(object):
diff --git a/test/jenkins_tests/run_asv.sh b/test/jenkins_tests/run_asv.sh
new file mode 100755
index 000000000..108826b0a
--- /dev/null
+++ b/test/jenkins_tests/run_asv.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# Cause the script to exit if a single command fails.
+set -e
+
+# Show explicitly which commands are currently running.
+set -x
+
+git clone https://github.com/ray-project/asv.git /tmp/asv/ || true
+cd /tmp/asv/
+pip install -e .
+
+cd /ray/python/
+asv machine --yes
+asv run --show-stderr --python=same --force-record-commit=$(cat ../git-rev)