[rllib] Use suppress_output instead of run_silent.sh script for tests (#4386)

* fix

* enable custom loss

* Update run_rllib_tests.sh

* enable tests

* fix action prob

* Update suppress_output

* fix example

* fix

.travis.yml

@@ -183,10 +183,10 @@ script:
   - if [ $RAY_CI_TUNE_AFFECTED == "1" ]; then python -m pytest --durations=10 --ignore=python/ray/tune/tests/test_cluster.py --ignore=python/ray/tune/tests/test_actor_reuse.py python/ray/tune/tests; fi
 
   # ray rllib tests
-  - if [ $RAY_CI_RLLIB_AFFECTED == "1" ]; then python/ray/rllib/tests/run_silent.sh tests/test_catalog.py; fi
-  - if [ $RAY_CI_RLLIB_AFFECTED == "1" ]; then python/ray/rllib/tests/run_silent.sh tests/test_filters.py; fi
-  - if [ $RAY_CI_RLLIB_AFFECTED == "1" ]; then python/ray/rllib/tests/run_silent.sh tests/test_optimizers.py; fi
-  - if [ $RAY_CI_RLLIB_AFFECTED == "1" ]; then python/ray/rllib/tests/run_silent.sh tests/test_evaluators.py; fi
+  - if [ $RAY_CI_RLLIB_AFFECTED == "1" ]; then ./ci/suppress_output python python/ray/rllib/tests/test_catalog.py; fi
+  - if [ $RAY_CI_RLLIB_AFFECTED == "1" ]; then ./ci/suppress_output python python/ray/rllib/tests/test_filters.py; fi
+  - if [ $RAY_CI_RLLIB_AFFECTED == "1" ]; then ./ci/suppress_output python python/ray/rllib/tests/test_optimizers.py; fi
+  - if [ $RAY_CI_RLLIB_AFFECTED == "1" ]; then ./ci/suppress_output python python/ray/rllib/tests/test_evaluators.py; fi
 
   # ray tests
   # Python3.5+ only. Otherwise we will get `SyntaxError` regardless of how we set the tester.

ci/jenkins_tests/run_rllib_tests.sh

@@ -1,47 +1,47 @@
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env PongDeterministic-v0 \
     --run A3C \
     --stop '{"training_iteration": 1}' \
     --config '{"num_workers": 2}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env Pong-ram-v4 \
     --run A3C \
     --stop '{"training_iteration": 1}' \
     --config '{"num_workers": 2}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env PongDeterministic-v0 \
     --run A2C \
     --stop '{"training_iteration": 1}' \
     --config '{"num_workers": 2}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v1 \
     --run PPO \
     --stop '{"training_iteration": 1}' \
     --config '{"kl_coeff": 1.0, "num_sgd_iter": 10, "lr": 1e-4, "sgd_minibatch_size": 64, "train_batch_size": 2000, "num_workers": 1, "model": {"free_log_std": true}}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v1 \
     --run PPO \
     --stop '{"training_iteration": 1}' \
     --config '{"simple_optimizer": false, "num_sgd_iter": 2, "model": {"use_lstm": true}}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v1 \
     --run PPO \
     --stop '{"training_iteration": 1}' \
     --config '{"simple_optimizer": true, "num_sgd_iter": 2, "model": {"use_lstm": true}}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v1 \
     --run PPO \
     --stop '{"training_iteration": 1}' \
@@ -49,208 +49,208 @@ docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
     --ray-num-gpus 1
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v1 \
     --run PPO \
     --stop '{"training_iteration": 1}' \
     --config '{"kl_coeff": 1.0, "num_sgd_iter": 10, "lr": 1e-4, "sgd_minibatch_size": 64, "train_batch_size": 2000, "num_workers": 1, "use_gae": false, "batch_mode": "complete_episodes"}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v1 \
     --run PPO \
     --stop '{"training_iteration": 1}' \
     --config '{"remote_worker_envs": true, "num_envs_per_worker": 2, "num_workers": 1, "train_batch_size": 100, "sgd_minibatch_size": 50}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v1 \
     --run PPO \
     --stop '{"training_iteration": 2}' \
     --config '{"async_remote_worker_envs": true, "num_envs_per_worker": 2, "num_workers": 1, "train_batch_size": 100, "sgd_minibatch_size": 50}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env Pendulum-v0 \
     --run APPO \
     --stop '{"training_iteration": 1}' \
     --config '{"num_workers": 2, "num_gpus": 0}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env Pendulum-v0 \
     --run ES \
     --stop '{"training_iteration": 1}' \
     --config '{"stepsize": 0.01, "episodes_per_batch": 20, "train_batch_size": 100, "num_workers": 2}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env Pong-v0 \
     --run ES \
     --stop '{"training_iteration": 1}' \
     --config '{"stepsize": 0.01, "episodes_per_batch": 20, "train_batch_size": 100, "num_workers": 2}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v0 \
     --run A3C \
     --stop '{"training_iteration": 1}' \
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v0 \
     --run DQN \
     --stop '{"training_iteration": 1}' \
     --config '{"lr": 1e-3, "schedule_max_timesteps": 100000, "exploration_fraction": 0.1, "exploration_final_eps": 0.02, "dueling": false, "hiddens": [], "model": {"fcnet_hiddens": [64], "fcnet_activation": "relu"}}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v0 \
     --run DQN \
     --stop '{"training_iteration": 1}' \
     --config '{"num_workers": 2}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v0 \
     --run APEX \
     --stop '{"training_iteration": 1}' \
     --config '{"num_workers": 2, "timesteps_per_iteration": 1000, "num_gpus": 0, "min_iter_time_s": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env FrozenLake-v0 \
     --run DQN \
     --stop '{"training_iteration": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env FrozenLake-v0 \
     --run PPO \
     --stop '{"training_iteration": 1}' \
     --config '{"num_sgd_iter": 10, "sgd_minibatch_size": 64, "train_batch_size": 1000, "num_workers": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env PongDeterministic-v4 \
     --run DQN \
     --stop '{"training_iteration": 1}' \
     --config '{"lr": 1e-4, "schedule_max_timesteps": 2000000, "buffer_size": 10000, "exploration_fraction": 0.1, "exploration_final_eps": 0.01, "sample_batch_size": 4, "learning_starts": 10000, "target_network_update_freq": 1000, "gamma": 0.99, "prioritized_replay": true}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env MontezumaRevenge-v0 \
     --run PPO \
     --stop '{"training_iteration": 1}' \
     --config '{"kl_coeff": 1.0, "num_sgd_iter": 10, "lr": 1e-4, "sgd_minibatch_size": 64, "train_batch_size": 2000, "num_workers": 1, "model": {"dim": 40, "conv_filters": [[16, [8, 8], 4], [32, [4, 4], 2], [512, [5, 5], 1]]}}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v1 \
     --run A3C \
     --stop '{"training_iteration": 1}' \
     --config '{"num_workers": 2, "model": {"use_lstm": true}}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v0 \
     --run DQN \
     --stop '{"training_iteration": 1}' \
     --config '{"num_workers": 2}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v0 \
     --run PG \
     --stop '{"training_iteration": 1}' \
     --config '{"sample_batch_size": 500, "num_workers": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v0 \
     --run PG \
     --stop '{"training_iteration": 1}' \
     --config '{"sample_batch_size": 500, "use_pytorch": true}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v0 \
     --run PG \
     --stop '{"training_iteration": 1}' \
     --config '{"sample_batch_size": 500, "num_workers": 1, "model": {"use_lstm": true, "max_seq_len": 100}}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v0 \
     --run PG \
     --stop '{"training_iteration": 1}' \
     --config '{"sample_batch_size": 500, "num_workers": 1, "num_envs_per_worker": 10}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env Pong-v0 \
     --run PG \
     --stop '{"training_iteration": 1}' \
     --config '{"sample_batch_size": 500, "num_workers": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env FrozenLake-v0 \
     --run PG \
     --stop '{"training_iteration": 1}' \
     --config '{"sample_batch_size": 500, "num_workers": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env Pendulum-v0 \
     --run DDPG \
     --stop '{"training_iteration": 1}' \
     --config '{"num_workers": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v0 \
     --run IMPALA \
     --stop '{"training_iteration": 1}' \
     --config '{"num_gpus": 0, "num_workers": 2, "min_iter_time_s": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v0 \
     --run IMPALA \
     --stop '{"training_iteration": 1}' \
     --config '{"num_gpus": 0, "num_workers": 2, "min_iter_time_s": 1, "model": {"use_lstm": true}}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v0 \
     --run IMPALA \
     --stop '{"training_iteration": 1}' \
     --config '{"num_gpus": 0, "num_workers": 2, "min_iter_time_s": 1, "num_data_loader_buffers": 2, "replay_buffer_num_slots": 100, "replay_proportion": 1.0}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v0 \
     --run IMPALA \
     --stop '{"training_iteration": 1}' \
     --config '{"num_gpus": 0, "num_workers": 2, "min_iter_time_s": 1, "num_data_loader_buffers": 2, "replay_buffer_num_slots": 100, "replay_proportion": 1.0, "model": {"use_lstm": true}}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env MountainCarContinuous-v0 \
     --run DDPG \
     --stop '{"training_iteration": 1}' \
     --config '{"num_workers": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env MountainCarContinuous-v0 \
     --run DDPG \
     --stop '{"training_iteration": 1}' \
     --config '{"num_workers": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env Pendulum-v0 \
     --run APEX_DDPG \
     --ray-num-cpus 8 \
@@ -258,153 +258,149 @@ docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
     --config '{"num_workers": 2, "optimizer": {"num_replay_buffer_shards": 1}, "learning_starts": 100, "min_iter_time_s": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env Pendulum-v0 \
     --run APEX_DDPG \
     --ray-num-cpus 8 \
     --stop '{"training_iteration": 1}' \
     --config '{"num_workers": 2, "optimizer": {"num_replay_buffer_shards": 1}, "learning_starts": 100, "min_iter_time_s": 1, "batch_mode": "complete_episodes", "parameter_noise": true}'
 
-# TODO(ericl): reenable the test after fix the arrow serialization error.
-# https://github.com/ray-project/ray/pull/4127#issuecomment-468903577
-#docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-#    /ray/python/ray/rllib/tests/run_silent.sh train.py \
-#    --env CartPole-v0 \
-#    --run MARWIL \
-#    --stop '{"training_iteration": 1}' \
-#    --config '{"input": "/ray/python/ray/rllib/tests/data/cartpole_small", "learning_starts": 0, "input_evaluation": ["wis", "is"], "shuffle_buffer_size": 10}'
-
-#docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-#    /ray/python/ray/rllib/tests/run_silent.sh train.py \
-#    --env CartPole-v0 \
-#    --run DQN \
-#    --stop '{"training_iteration": 1}' \
-#    --config '{"input": "/ray/python/ray/rllib/tests/data/cartpole_small", "learning_starts": 0, "input_evaluation": ["wis", "is"], "soft_q": true}'
+docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
+    --env CartPole-v0 \
+    --run MARWIL \
+    --stop '{"training_iteration": 1}' \
+    --config '{"input": "/ray/python/ray/rllib/tests/data/cartpole_small", "learning_starts": 0, "input_evaluation": ["wis", "is"], "shuffle_buffer_size": 10}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh tests/test_local.py
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
+    --env CartPole-v0 \
+    --run DQN \
+    --stop '{"training_iteration": 1}' \
+    --config '{"input": "/ray/python/ray/rllib/tests/data/cartpole_small", "learning_starts": 0, "input_evaluation": ["wis", "is"], "soft_q": true}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh tests/test_io.py
+    /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_local.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh tests/test_checkpoint_restore.py
+    /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_io.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh tests/test_policy_evaluator.py
+    /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_checkpoint_restore.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh tests/test_nested_spaces.py
+    /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_policy_evaluator.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh tests/test_external_env.py
+    /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_nested_spaces.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh examples/parametric_action_cartpole.py --run=PG --stop=50
+    /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_external_env.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh examples/parametric_action_cartpole.py --run=PPO --stop=50
+    /ray/ci/suppress_output python /ray/python/ray/rllib/examples/parametric_action_cartpole.py --run=PG --stop=50
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh examples/parametric_action_cartpole.py --run=DQN --stop=50
+    /ray/ci/suppress_output python /ray/python/ray/rllib/examples/parametric_action_cartpole.py --run=PPO --stop=50
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh tests/test_lstm.py
+    /ray/ci/suppress_output python /ray/python/ray/rllib/examples/parametric_action_cartpole.py --run=DQN --stop=50
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh examples/batch_norm_model.py --num-iters=1 --run=PPO
+    /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_lstm.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh examples/batch_norm_model.py --num-iters=1 --run=PG
+    /ray/ci/suppress_output python /ray/python/ray/rllib/examples/batch_norm_model.py --num-iters=1 --run=PPO
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh examples/batch_norm_model.py --num-iters=1 --run=DQN
+    /ray/ci/suppress_output python /ray/python/ray/rllib/examples/batch_norm_model.py --num-iters=1 --run=PG
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh examples/batch_norm_model.py --num-iters=1 --run=DDPG
+    /ray/ci/suppress_output python /ray/python/ray/rllib/examples/batch_norm_model.py --num-iters=1 --run=DQN
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh tests/test_multi_agent_env.py
+    /ray/ci/suppress_output python /ray/python/ray/rllib/examples/batch_norm_model.py --num-iters=1 --run=DDPG
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh tests/test_supported_spaces.py
+    /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_multi_agent_env.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh tests/test_env_with_subprocess.py
+    /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_supported_spaces.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh tests/test_rollout.sh
+    /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_env_with_subprocess.py
+
+docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
+    /ray/ci/suppress_output /ray/python/ray/rllib/tests/test_rollout.sh
 
 # Run all single-agent regression tests (3x retry each)
 for yaml in $(ls $ROOT_DIR/../../python/ray/rllib/tuned_examples/regression_tests); do
     docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-        /ray/python/ray/rllib/tests/run_silent.sh tests/run_regression_tests.py \
+        /ray/ci/suppress_output python /ray/python/ray/rllib/tests/run_regression_tests.py \
             /ray/python/ray/rllib/tuned_examples/regression_tests/$yaml
 done
 
 # Try a couple times since it's stochastic
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-        /ray/python/ray/rllib/tests/run_silent.sh tests/multiagent_pendulum.py || \
+        /ray/ci/suppress_output python /ray/python/ray/rllib/tests/multiagent_pendulum.py || \
     docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-        /ray/python/ray/rllib/tests/run_silent.sh tests/multiagent_pendulum.py || \
+        /ray/ci/suppress_output python /ray/python/ray/rllib/tests/multiagent_pendulum.py || \
     docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-        /ray/python/ray/rllib/tests/run_silent.sh tests/multiagent_pendulum.py
+        /ray/ci/suppress_output python /ray/python/ray/rllib/tests/multiagent_pendulum.py
 
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh examples/multiagent_cartpole.py --num-iters=2
+    /ray/ci/suppress_output python /ray/python/ray/rllib/examples/multiagent_cartpole.py --num-iters=2
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh examples/multiagent_two_trainers.py --num-iters=2
+    /ray/ci/suppress_output python /ray/python/ray/rllib/examples/multiagent_two_trainers.py --num-iters=2
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh tests/test_avail_actions_qmix.py
+    /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_avail_actions_qmix.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh examples/cartpole_lstm.py --run=PPO --stop=200
+    /ray/ci/suppress_output python /ray/python/ray/rllib/examples/cartpole_lstm.py --run=PPO --stop=200
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh examples/cartpole_lstm.py --run=IMPALA --stop=100
+    /ray/ci/suppress_output python /ray/python/ray/rllib/examples/cartpole_lstm.py --run=IMPALA --stop=100
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh examples/cartpole_lstm.py --stop=200 --use-prev-action-reward
-
-# TODO(ericl): reenable the test after fix the arrow serialization error.
-# https://github.com/ray-project/ray/pull/4127#issuecomment-468903577
-#docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-#    /ray/python/ray/rllib/tests/run_silent.sh examples/custom_loss.py --iters=2
+    /ray/ci/suppress_output python /ray/python/ray/rllib/examples/cartpole_lstm.py --stop=200 --use-prev-action-reward
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh examples/custom_metrics_and_callbacks.py --num-iters=2
+    /ray/ci/suppress_output python /ray/python/ray/rllib/examples/custom_loss.py --iters=2
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh contrib/random_agent/random_agent.py
+    /ray/ci/suppress_output python /ray/python/ray/rllib/examples/custom_metrics_and_callbacks.py --num-iters=2
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh examples/twostep_game.py --stop=2000 --run=PG
+    /ray/ci/suppress_output python /ray/python/ray/rllib/contrib/random_agent/random_agent.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh examples/twostep_game.py --stop=2000 --run=QMIX
+    /ray/ci/suppress_output python /ray/python/ray/rllib/examples/twostep_game.py --stop=2000 --run=PG
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh examples/twostep_game.py --stop=2000 --run=APEX_QMIX
+    /ray/ci/suppress_output python /ray/python/ray/rllib/examples/twostep_game.py --stop=2000 --run=QMIX
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output python /ray/python/ray/rllib/examples/twostep_game.py --stop=2000 --run=APEX_QMIX
+
+docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env PongDeterministic-v4 \
     --run A3C \
     --stop '{"training_iteration": 1}' \
     --config '{"num_workers": 2, "use_pytorch": true, "sample_async": false, "model": {"use_lstm": false, "grayscale": true, "zero_mean": false, "dim": 84}, "preprocessor_pref": "rllib"}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env CartPole-v1 \
     --run A3C \
     --stop '{"training_iteration": 1}' \
     --config '{"num_workers": 2, "use_pytorch": true, "sample_async": false}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh train.py \
+    /ray/ci/suppress_output /ray/python/ray/rllib/train.py \
     --env PongDeterministic-v4 \
     --run IMPALA \
     --stop='{"timesteps_total": 40000}' \
@@ -412,7 +408,7 @@ docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
     --config '{"num_workers": 1, "num_gpus": 0, "num_envs_per_worker": 64, "sample_batch_size": 50, "train_batch_size": 50, "learner_queue_size": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh agents/impala/vtrace_test.py
+    /ray/ci/suppress_output python /ray/python/ray/rllib/agents/impala/vtrace_test.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/tests/run_silent.sh tests/test_ignore_worker_failure.py
+    /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_ignore_worker_failure.py

ci/suppress_output

@@ -2,7 +2,6 @@
 # Run a command, suppressing output unless it hangs or crashes.
 
 TMPFILE=`mktemp`
-COMMAND="$@"
 PID=$$
 
 # Print output to avoid travis killing us
@@ -20,7 +19,7 @@ watchdog() {
 watchdog & 2>/dev/null
 WATCHDOG_PID=$!
 
-time $COMMAND >$TMPFILE 2>&1
+time "$@" >$TMPFILE 2>&1
 
 CODE=$?
 if [ $CODE != 0 ]; then

python/ray/rllib/examples/custom_loss.py

@@ -51,9 +51,9 @@ class CustomLossModel(Model):
         input_ops = reader.tf_input_ops()
 
         # define a secondary loss by building a graph copy with weight sharing
+        obs = tf.cast(input_ops["obs"], tf.float32)
         logits, _ = self._build_layers_v2({
-            "obs": restore_original_dimensions(input_ops["obs"],
-                                               self.obs_space)
+            "obs": restore_original_dimensions(obs, self.obs_space)
         }, self.num_outputs, self.options)
 
         # You can also add self-supervised losses easily by referencing tensors

python/ray/rllib/offline/off_policy_estimator.py

@@ -92,8 +92,9 @@ class OffPolicyEstimator(object):
             raise ValueError(
                 "Off-policy estimation is not possible unless the inputs "
                 "include action probabilities (i.e., the policy is stochastic "
-                "and emits the 'action_prob' key). You can set "
-                "`input_evaluation: []` to resolve this.")
+                "and emits the 'action_prob' key). For DQN this means using "
+                "`soft_q: True`. You can also set `input_evaluation: []` to "
+                "disable estimation.")
 
     @DeveloperAPI
     def get_metrics(self):

python/ray/rllib/tests/run_silent.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-TMPFILE=`mktemp`
-DIRECTORY=`dirname $0`
-SCRIPT=$1
-shift
-
-if [ -x $DIRECTORY/../$SCRIPT ]; then
-    time $DIRECTORY/../$SCRIPT "$@" >$TMPFILE 2>&1
-else
-    time python $DIRECTORY/../$SCRIPT "$@" >$TMPFILE 2>&1
-fi
-
-CODE=$?
-if [ $CODE != 0 ]; then
-    cat $TMPFILE
-    echo "FAILED $CODE"
-    exit $CODE
-fi
-
-exit 0