diff --git a/python/ray/rllib/optimizers/sync_replay_optimizer.py b/python/ray/rllib/optimizers/sync_replay_optimizer.py
index 21f00d7df..366be6696 100644
--- a/python/ray/rllib/optimizers/sync_replay_optimizer.py
+++ b/python/ray/rllib/optimizers/sync_replay_optimizer.py
@@ -2,6 +2,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import logging
 import collections
 import numpy as np
 
@@ -17,6 +18,8 @@ from ray.rllib.utils.compression import pack_if_needed
 from ray.rllib.utils.timer import TimerStat
 from ray.rllib.utils.schedules import LinearSchedule
 
+logger = logging.getLogger(__name__)
+
 
 class SyncReplayOptimizer(PolicyOptimizer):
     """Variant of the local sync optimizer that supports replay (for DQN).
@@ -69,7 +72,9 @@ class SyncReplayOptimizer(PolicyOptimizer):
 
         self.replay_buffers = collections.defaultdict(new_buffer)
 
-        assert buffer_size >= self.replay_starts
+        if buffer_size < self.replay_starts:
+            logger.warning("buffer_size={} < replay_starts={}".format(
+                buffer_size, self.replay_starts))
 
     @override(PolicyOptimizer)
     def step(self):