diff --git a/rllib/agents/ppo/ppo.py b/rllib/agents/ppo/ppo.py
index 3df44b412..828402603 100644
--- a/rllib/agents/ppo/ppo.py
+++ b/rllib/agents/ppo/ppo.py
@@ -129,6 +129,8 @@ def warn_about_bad_reward_scales(trainer, result):
 def validate_config(config):
     if config["entropy_coeff"] < 0:
         raise DeprecationWarning("entropy_coeff must be >= 0")
+    if isinstance(config["entropy_coeff"], int):
+        config["entropy_coeff"] = float(config["entropy_coeff"])
     if config["sgd_minibatch_size"] > config["train_batch_size"]:
         raise ValueError(
             "Minibatch size {} must be <= train batch size {}.".format(