diff --git a/.travis.yml b/.travis.yml
index ab8b6a333..7303013db 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -85,6 +85,8 @@ matrix:
         # except the `$MOUNT_BAZEL_CACHE` part.
 
         - ./ci/suppress_output docker run --rm -w /ray -v `pwd`:/ray $MOUNT_BAZEL_CACHE -ti rayproject/arrow_linux_x86_64_base:latest /ray/python/build-wheel-manylinux1.sh
+
+        - $TRAVIS_BUILD_DIR/ci/travis/build-autoscaler-images.sh
       script:
         - if [ $RAY_CI_LINUX_WHEELS_AFFECTED != "1" ]; then exit; fi
 
@@ -165,12 +167,16 @@ deploy:
     local_dir: .whl
     upload-dir: latest
     skip_cleanup: true
-    only:
-      - master
     on:
+      branch: master
       repo: ray-project/ray
       condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1
 
 cache:
   directories:
     - $HOME/ray-bazel-cache
+
+env:
+  global:
+    # DOCKER_PASSWORD
+    secure: "I74yutR4HoiAQna9Ywi7uHcQDoefA+9lm5u24Xuu9Gha8Q7Anvic0qg4ILFJdqX+N7IdXtPA+/Xj+rjBNr7xNtamN1TmCItlXYtl1AMZ35bQZJjaJD3Ath/xe6Cj82BsBXsQYDwIXa8f8J0AsDNDW+u0+YysA7R86xQq086snevKttKXHBYfzzB2PLN6CBQh2RGWanE0qR2SOVSg+F0Ra+epZJ6jaey90UL4t6llbsNIlR17Xqc9OtFjDRIvO1ohpC7OcqnVDkEQcckue0CIHFwJ1J4kFzoJ31NFG/ZqNRzaoBtb8xiVkVQnxHRmX0ZRgx8z6EPfVMVBf4HBxgqUNgSnumV2ljoJV1/N8Ho+JkvnfzhLbdacG2qdKHQex85xnS8esVl1uvmZgJzPyX9u3OcLwNnzuFP3+CcJETsH19mwDo8dZzMy/jYzx+26oM410umG6KIMf1rvInDb2FEDT3lbUFdRMteDC5fjq6H/z7AHL2LDaS0pfEO9p5FRo8xokk7FT4row6yZbnAW/lqT52P8g1VNxrXzMWnzN5OeFhC7VaeJC6PaVG1eVwbgjRnoOhfXqwX5gms7KE03g7j1vXwbf4nbg53SJfwPFcvzW3az9G892jy170tsEzAbRVJKD14rsRQUh1lTq75GlEYEWvB1H2joUNtEgok0XExiTbY="
diff --git a/ci/travis/build-autoscaler-images.sh b/ci/travis/build-autoscaler-images.sh
new file mode 100755
index 000000000..2d33a7ae7
--- /dev/null
+++ b/ci/travis/build-autoscaler-images.sh
@@ -0,0 +1,36 @@
+# This script build docker images for autoscaler.
+# For now, we only build python3.6 images.
+set -e
+set -x
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+ROOT_DIR=$(cd $SCRIPT_DIR/../../; pwd)
+DOCKER_USERNAME="raytravisbot"
+
+# We will only build and push when we are building branch build.
+if [[ "$TRAVIS" == "true" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then
+    echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+
+    docker build -q -t ray-project/base-deps docker/base-deps
+
+    wheel="ray-0.8.0.dev6-cp36-cp36m-manylinux1_x86_64.whl"
+    commit_sha=$(echo $TRAVIS_COMMIT | head -c 6)
+    cp -r $ROOT_DIR/.whl $ROOT_DIR/docker/autoscaler/.whl
+
+    docker build \
+        --build-arg WHEEL_PATH=".whl/$wheel" \
+        --build-arg WHEEL_NAME=$wheel \
+        -t ray-project/autoscaler:$commit_sha \
+        $ROOT_DIR/docker/autoscaler
+    docker push ray-project/autoscaler:$commit_sha
+
+    # We have a branch build, e.g. release/v0.7.0
+    if [[ "$TRAVIS_BRANCH" != "master" ]] then
+       docker tag ray-project/autoscaler:$commit_sha ray-project/autoscaler:$TRAVIS_BRANCH
+       docker push ray-project/autoscaler:$TRAVIS_BRANCH
+    else
+       docker tag ray-project/autoscaler:$commit_sha ray-project/autoscaler:latest
+       docker push ray-project/autoscaler:latest
+    fi
+fi
+
diff --git a/docker/autoscaler/Dockerfile b/docker/autoscaler/Dockerfile
new file mode 100644
index 000000000..ad1470f99
--- /dev/null
+++ b/docker/autoscaler/Dockerfile
@@ -0,0 +1,18 @@
+FROM ray-project/base-deps
+ARG WHEEL_PATH
+ARG WHEEL_NAME
+
+RUN apt update
+RUN apt install -y curl tmux screen rsync
+
+# We have to uninstall wrapt this way for Tensorflow compatibility
+RUN conda remove -y wrapt
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
+COPY $WHEEL_PATH $WHEEL_NAME
+RUN pip install $WHEEL_NAME[all]
+
+# For Click
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
diff --git a/docker/autoscaler/requirements.txt b/docker/autoscaler/requirements.txt
new file mode 100644
index 000000000..0e6e554ac
--- /dev/null
+++ b/docker/autoscaler/requirements.txt
@@ -0,0 +1,24 @@
+gym[atari]
+opencv-python-headless
+tensorflow
+lz4
+keras
+smart_open
+tensorflow_probability
+h5py
+bayesian-optimization
+hyperopt==0.1.2
+ConfigSpace==0.4.10
+sigopt
+nevergrad 
+scikit-optimize 
+hpbandster 
+lightgbm 
+xgboost 
+torch 
+torchvision 
+tensorboardX
+tabulate
+mlflow
+kubernetes
+boto3
diff --git a/python/setup.py b/python/setup.py
index c65231fce..b43b56e09 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -3,6 +3,7 @@ from __future__ import division
 from __future__ import print_function
 
 import glob
+from itertools import chain
 import os
 import re
 import shutil
@@ -82,6 +83,8 @@ extras = {
     "tune": ["tabulate"],
 }
 
+extras["all"] = list(set(chain.from_iterable(extras.values())))
+
 
 class build_ext(_build_ext.build_ext):
     def run(self):