diff --git a/.travis.yml b/.travis.yml
index 9e070a194..7cf3d7ea3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -98,9 +98,13 @@ matrix:
         - ./ci/travis/install-dependencies.sh
         - export PATH="$HOME/miniconda/bin:$PATH"
         - cd doc
+        # readthedocs has an antiquated build env.
+        # This is a best effort to reproduce it locally to avoid doc build failures
+        # and hidden errors.
+        - pip install -r requirements-rtd.txt
         - pip install -r requirements-doc.txt
         - pip install yapf==0.23.0
-        - sphinx-build -q -W -b html -d _build/doctrees source _build/html
+        - sphinx-build -q -W -E -T -b html source _build/html
         - cd ..
         # Run Python linting, ignore dict vs {} (C408), others are defaults
         - flake8 --inline-quotes '"' --no-avoid-escape --exclude=python/ray/core/generated/,streaming/python/generated,doc/source/conf.py,python/ray/cloudpickle/,python/ray/thirdparty_files --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605
diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt
index 3066e2d8b..59ad03acf 100644
--- a/doc/requirements-doc.txt
+++ b/doc/requirements-doc.txt
@@ -15,7 +15,7 @@ pygments
 pyyaml
 recommonmark
 redis
-sphinx
+sphinx>2
 sphinx-click
 sphinx-copybutton
 sphinx-gallery
diff --git a/doc/requirements-rtd.txt b/doc/requirements-rtd.txt
new file mode 100644
index 000000000..5245aa83a
--- /dev/null
+++ b/doc/requirements-rtd.txt
@@ -0,0 +1,11 @@
+Pygments==2.3.1
+setuptools==41.0.1
+docutils==0.14
+mock==1.0.1
+pillow==5.4.1
+alabaster>=0.7,<0.8,!=0.7.5
+commonmark==0.8.1
+recommonmark==0.5.0
+sphinx<2
+sphinx-rtd-theme<0.5
+readthedocs-sphinx-ext<1.1
diff --git a/doc/source/conf.py b/doc/source/conf.py
index 6743157ff..4c43db666 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -27,7 +27,7 @@ MOCK_MODULES = [
     "ray.core.generated", "ray.core.generated.gcs_pb2",
     "ray.core.generated.ray.protocol.Task", "scipy", "scipy.signal",
     "scipy.stats", "setproctitle", "tensorflow_probability", "tensorflow",
-    "tensorflow.contrib", "tensorflow.contrib.all_reduce",
+    "tensorflow.contrib", "tensorflow.contrib.all_reduce", "tree",
     "tensorflow.contrib.all_reduce.python", "tensorflow.contrib.layers",
     "tensorflow.contrib.rnn", "tensorflow.contrib.slim", "tensorflow.core",
     "tensorflow.core.util", "tensorflow.python", "tensorflow.python.client",
diff --git a/python/ray/util/sgd/torch/training_operator.py b/python/ray/util/sgd/torch/training_operator.py
index a35a4886a..873767dc1 100644
--- a/python/ray/util/sgd/torch/training_operator.py
+++ b/python/ray/util/sgd/torch/training_operator.py
@@ -1,6 +1,4 @@
 import collections
-
-from tqdm import tqdm
 import torch
 
 from ray.util.sgd.utils import (TimerCollection, AverageMeterCollection,
@@ -18,6 +16,11 @@ except ImportError:
     # where amp is initialized.
     pass
 
+tqdm = None
+try:
+    from tqdm import tqdm
+except ImportError:
+    pass
 
 def _is_multiple(component):
     """Checks if a component (optimizer, model, etc) is not singular."""
@@ -76,6 +79,8 @@ class TrainingOperator:
                     type(schedulers)))
         self._config = config
         self._use_fp16 = use_fp16
+        if tqdm is None and use_tqdm:
+            raise ValueError("tqdm must be installed to use tqdm in training.")
         self._use_tqdm = use_tqdm
         self.global_step = 0