From 57c1aeb427fa116790a16c17777ce944ccb5666c Mon Sep 17 00:00:00 2001 From: Eric Liang Date: Thu, 21 Mar 2019 00:15:24 -0700 Subject: [PATCH] [rllib] Use suppress_output instead of run_silent.sh script for tests (#4386) * fix * enable custom loss * Update run_rllib_tests.sh * enable tests * fix action prob * Update suppress_output * fix example * fix --- .travis.yml | 8 +- ci/jenkins_tests/run_rllib_tests.sh | 188 +++++++++--------- ci/suppress_output | 3 +- python/ray/rllib/examples/custom_loss.py | 4 +- .../ray/rllib/offline/off_policy_estimator.py | 5 +- ...output-2019-02-03_20-27-20_worker-0_0.json | 7 +- python/ray/rllib/tests/run_silent.sh | 21 -- 7 files changed, 105 insertions(+), 131 deletions(-) delete mode 100755 python/ray/rllib/tests/run_silent.sh diff --git a/.travis.yml b/.travis.yml index 4504f99df..618236f7e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -183,10 +183,10 @@ script: - if [ $RAY_CI_TUNE_AFFECTED == "1" ]; then python -m pytest --durations=10 --ignore=python/ray/tune/tests/test_cluster.py --ignore=python/ray/tune/tests/test_actor_reuse.py python/ray/tune/tests; fi # ray rllib tests - - if [ $RAY_CI_RLLIB_AFFECTED == "1" ]; then python/ray/rllib/tests/run_silent.sh tests/test_catalog.py; fi - - if [ $RAY_CI_RLLIB_AFFECTED == "1" ]; then python/ray/rllib/tests/run_silent.sh tests/test_filters.py; fi - - if [ $RAY_CI_RLLIB_AFFECTED == "1" ]; then python/ray/rllib/tests/run_silent.sh tests/test_optimizers.py; fi - - if [ $RAY_CI_RLLIB_AFFECTED == "1" ]; then python/ray/rllib/tests/run_silent.sh tests/test_evaluators.py; fi + - if [ $RAY_CI_RLLIB_AFFECTED == "1" ]; then ./ci/suppress_output python python/ray/rllib/tests/test_catalog.py; fi + - if [ $RAY_CI_RLLIB_AFFECTED == "1" ]; then ./ci/suppress_output python python/ray/rllib/tests/test_filters.py; fi + - if [ $RAY_CI_RLLIB_AFFECTED == "1" ]; then ./ci/suppress_output python python/ray/rllib/tests/test_optimizers.py; fi + - if [ $RAY_CI_RLLIB_AFFECTED == "1" ]; then ./ci/suppress_output python python/ray/rllib/tests/test_evaluators.py; fi # ray tests # Python3.5+ only. Otherwise we will get `SyntaxError` regardless of how we set the tester. diff --git a/ci/jenkins_tests/run_rllib_tests.sh b/ci/jenkins_tests/run_rllib_tests.sh index fda03cfb2..f6c88811a 100644 --- a/ci/jenkins_tests/run_rllib_tests.sh +++ b/ci/jenkins_tests/run_rllib_tests.sh @@ -1,47 +1,47 @@ docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env PongDeterministic-v0 \ --run A3C \ --stop '{"training_iteration": 1}' \ --config '{"num_workers": 2}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env Pong-ram-v4 \ --run A3C \ --stop '{"training_iteration": 1}' \ --config '{"num_workers": 2}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env PongDeterministic-v0 \ --run A2C \ --stop '{"training_iteration": 1}' \ --config '{"num_workers": 2}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v1 \ --run PPO \ --stop '{"training_iteration": 1}' \ --config '{"kl_coeff": 1.0, "num_sgd_iter": 10, "lr": 1e-4, "sgd_minibatch_size": 64, "train_batch_size": 2000, "num_workers": 1, "model": {"free_log_std": true}}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v1 \ --run PPO \ --stop '{"training_iteration": 1}' \ --config '{"simple_optimizer": false, "num_sgd_iter": 2, "model": {"use_lstm": true}}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v1 \ --run PPO \ --stop '{"training_iteration": 1}' \ --config '{"simple_optimizer": true, "num_sgd_iter": 2, "model": {"use_lstm": true}}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v1 \ --run PPO \ --stop '{"training_iteration": 1}' \ @@ -49,208 +49,208 @@ docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ --ray-num-gpus 1 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v1 \ --run PPO \ --stop '{"training_iteration": 1}' \ --config '{"kl_coeff": 1.0, "num_sgd_iter": 10, "lr": 1e-4, "sgd_minibatch_size": 64, "train_batch_size": 2000, "num_workers": 1, "use_gae": false, "batch_mode": "complete_episodes"}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v1 \ --run PPO \ --stop '{"training_iteration": 1}' \ --config '{"remote_worker_envs": true, "num_envs_per_worker": 2, "num_workers": 1, "train_batch_size": 100, "sgd_minibatch_size": 50}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v1 \ --run PPO \ --stop '{"training_iteration": 2}' \ --config '{"async_remote_worker_envs": true, "num_envs_per_worker": 2, "num_workers": 1, "train_batch_size": 100, "sgd_minibatch_size": 50}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env Pendulum-v0 \ --run APPO \ --stop '{"training_iteration": 1}' \ --config '{"num_workers": 2, "num_gpus": 0}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env Pendulum-v0 \ --run ES \ --stop '{"training_iteration": 1}' \ --config '{"stepsize": 0.01, "episodes_per_batch": 20, "train_batch_size": 100, "num_workers": 2}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env Pong-v0 \ --run ES \ --stop '{"training_iteration": 1}' \ --config '{"stepsize": 0.01, "episodes_per_batch": 20, "train_batch_size": 100, "num_workers": 2}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v0 \ --run A3C \ --stop '{"training_iteration": 1}' \ docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v0 \ --run DQN \ --stop '{"training_iteration": 1}' \ --config '{"lr": 1e-3, "schedule_max_timesteps": 100000, "exploration_fraction": 0.1, "exploration_final_eps": 0.02, "dueling": false, "hiddens": [], "model": {"fcnet_hiddens": [64], "fcnet_activation": "relu"}}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v0 \ --run DQN \ --stop '{"training_iteration": 1}' \ --config '{"num_workers": 2}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v0 \ --run APEX \ --stop '{"training_iteration": 1}' \ --config '{"num_workers": 2, "timesteps_per_iteration": 1000, "num_gpus": 0, "min_iter_time_s": 1}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env FrozenLake-v0 \ --run DQN \ --stop '{"training_iteration": 1}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env FrozenLake-v0 \ --run PPO \ --stop '{"training_iteration": 1}' \ --config '{"num_sgd_iter": 10, "sgd_minibatch_size": 64, "train_batch_size": 1000, "num_workers": 1}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env PongDeterministic-v4 \ --run DQN \ --stop '{"training_iteration": 1}' \ --config '{"lr": 1e-4, "schedule_max_timesteps": 2000000, "buffer_size": 10000, "exploration_fraction": 0.1, "exploration_final_eps": 0.01, "sample_batch_size": 4, "learning_starts": 10000, "target_network_update_freq": 1000, "gamma": 0.99, "prioritized_replay": true}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env MontezumaRevenge-v0 \ --run PPO \ --stop '{"training_iteration": 1}' \ --config '{"kl_coeff": 1.0, "num_sgd_iter": 10, "lr": 1e-4, "sgd_minibatch_size": 64, "train_batch_size": 2000, "num_workers": 1, "model": {"dim": 40, "conv_filters": [[16, [8, 8], 4], [32, [4, 4], 2], [512, [5, 5], 1]]}}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v1 \ --run A3C \ --stop '{"training_iteration": 1}' \ --config '{"num_workers": 2, "model": {"use_lstm": true}}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v0 \ --run DQN \ --stop '{"training_iteration": 1}' \ --config '{"num_workers": 2}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v0 \ --run PG \ --stop '{"training_iteration": 1}' \ --config '{"sample_batch_size": 500, "num_workers": 1}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v0 \ --run PG \ --stop '{"training_iteration": 1}' \ --config '{"sample_batch_size": 500, "use_pytorch": true}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v0 \ --run PG \ --stop '{"training_iteration": 1}' \ --config '{"sample_batch_size": 500, "num_workers": 1, "model": {"use_lstm": true, "max_seq_len": 100}}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v0 \ --run PG \ --stop '{"training_iteration": 1}' \ --config '{"sample_batch_size": 500, "num_workers": 1, "num_envs_per_worker": 10}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env Pong-v0 \ --run PG \ --stop '{"training_iteration": 1}' \ --config '{"sample_batch_size": 500, "num_workers": 1}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env FrozenLake-v0 \ --run PG \ --stop '{"training_iteration": 1}' \ --config '{"sample_batch_size": 500, "num_workers": 1}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env Pendulum-v0 \ --run DDPG \ --stop '{"training_iteration": 1}' \ --config '{"num_workers": 1}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v0 \ --run IMPALA \ --stop '{"training_iteration": 1}' \ --config '{"num_gpus": 0, "num_workers": 2, "min_iter_time_s": 1}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v0 \ --run IMPALA \ --stop '{"training_iteration": 1}' \ --config '{"num_gpus": 0, "num_workers": 2, "min_iter_time_s": 1, "model": {"use_lstm": true}}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v0 \ --run IMPALA \ --stop '{"training_iteration": 1}' \ --config '{"num_gpus": 0, "num_workers": 2, "min_iter_time_s": 1, "num_data_loader_buffers": 2, "replay_buffer_num_slots": 100, "replay_proportion": 1.0}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v0 \ --run IMPALA \ --stop '{"training_iteration": 1}' \ --config '{"num_gpus": 0, "num_workers": 2, "min_iter_time_s": 1, "num_data_loader_buffers": 2, "replay_buffer_num_slots": 100, "replay_proportion": 1.0, "model": {"use_lstm": true}}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env MountainCarContinuous-v0 \ --run DDPG \ --stop '{"training_iteration": 1}' \ --config '{"num_workers": 1}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env MountainCarContinuous-v0 \ --run DDPG \ --stop '{"training_iteration": 1}' \ --config '{"num_workers": 1}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env Pendulum-v0 \ --run APEX_DDPG \ --ray-num-cpus 8 \ @@ -258,153 +258,149 @@ docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ --config '{"num_workers": 2, "optimizer": {"num_replay_buffer_shards": 1}, "learning_starts": 100, "min_iter_time_s": 1}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env Pendulum-v0 \ --run APEX_DDPG \ --ray-num-cpus 8 \ --stop '{"training_iteration": 1}' \ --config '{"num_workers": 2, "optimizer": {"num_replay_buffer_shards": 1}, "learning_starts": 100, "min_iter_time_s": 1, "batch_mode": "complete_episodes", "parameter_noise": true}' -# TODO(ericl): reenable the test after fix the arrow serialization error. -# https://github.com/ray-project/ray/pull/4127#issuecomment-468903577 -#docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ -# /ray/python/ray/rllib/tests/run_silent.sh train.py \ -# --env CartPole-v0 \ -# --run MARWIL \ -# --stop '{"training_iteration": 1}' \ -# --config '{"input": "/ray/python/ray/rllib/tests/data/cartpole_small", "learning_starts": 0, "input_evaluation": ["wis", "is"], "shuffle_buffer_size": 10}' - -#docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ -# /ray/python/ray/rllib/tests/run_silent.sh train.py \ -# --env CartPole-v0 \ -# --run DQN \ -# --stop '{"training_iteration": 1}' \ -# --config '{"input": "/ray/python/ray/rllib/tests/data/cartpole_small", "learning_starts": 0, "input_evaluation": ["wis", "is"], "soft_q": true}' +docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ + --env CartPole-v0 \ + --run MARWIL \ + --stop '{"training_iteration": 1}' \ + --config '{"input": "/ray/python/ray/rllib/tests/data/cartpole_small", "learning_starts": 0, "input_evaluation": ["wis", "is"], "shuffle_buffer_size": 10}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh tests/test_local.py + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ + --env CartPole-v0 \ + --run DQN \ + --stop '{"training_iteration": 1}' \ + --config '{"input": "/ray/python/ray/rllib/tests/data/cartpole_small", "learning_starts": 0, "input_evaluation": ["wis", "is"], "soft_q": true}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh tests/test_io.py + /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_local.py docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh tests/test_checkpoint_restore.py + /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_io.py docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh tests/test_policy_evaluator.py + /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_checkpoint_restore.py docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh tests/test_nested_spaces.py + /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_policy_evaluator.py docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh tests/test_external_env.py + /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_nested_spaces.py docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh examples/parametric_action_cartpole.py --run=PG --stop=50 + /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_external_env.py docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh examples/parametric_action_cartpole.py --run=PPO --stop=50 + /ray/ci/suppress_output python /ray/python/ray/rllib/examples/parametric_action_cartpole.py --run=PG --stop=50 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh examples/parametric_action_cartpole.py --run=DQN --stop=50 + /ray/ci/suppress_output python /ray/python/ray/rllib/examples/parametric_action_cartpole.py --run=PPO --stop=50 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh tests/test_lstm.py + /ray/ci/suppress_output python /ray/python/ray/rllib/examples/parametric_action_cartpole.py --run=DQN --stop=50 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh examples/batch_norm_model.py --num-iters=1 --run=PPO + /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_lstm.py docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh examples/batch_norm_model.py --num-iters=1 --run=PG + /ray/ci/suppress_output python /ray/python/ray/rllib/examples/batch_norm_model.py --num-iters=1 --run=PPO docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh examples/batch_norm_model.py --num-iters=1 --run=DQN + /ray/ci/suppress_output python /ray/python/ray/rllib/examples/batch_norm_model.py --num-iters=1 --run=PG docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh examples/batch_norm_model.py --num-iters=1 --run=DDPG + /ray/ci/suppress_output python /ray/python/ray/rllib/examples/batch_norm_model.py --num-iters=1 --run=DQN docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh tests/test_multi_agent_env.py + /ray/ci/suppress_output python /ray/python/ray/rllib/examples/batch_norm_model.py --num-iters=1 --run=DDPG docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh tests/test_supported_spaces.py + /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_multi_agent_env.py docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh tests/test_env_with_subprocess.py + /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_supported_spaces.py docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh tests/test_rollout.sh + /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_env_with_subprocess.py + +docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ + /ray/ci/suppress_output /ray/python/ray/rllib/tests/test_rollout.sh # Run all single-agent regression tests (3x retry each) for yaml in $(ls $ROOT_DIR/../../python/ray/rllib/tuned_examples/regression_tests); do docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh tests/run_regression_tests.py \ + /ray/ci/suppress_output python /ray/python/ray/rllib/tests/run_regression_tests.py \ /ray/python/ray/rllib/tuned_examples/regression_tests/$yaml done # Try a couple times since it's stochastic docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh tests/multiagent_pendulum.py || \ + /ray/ci/suppress_output python /ray/python/ray/rllib/tests/multiagent_pendulum.py || \ docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh tests/multiagent_pendulum.py || \ + /ray/ci/suppress_output python /ray/python/ray/rllib/tests/multiagent_pendulum.py || \ docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh tests/multiagent_pendulum.py + /ray/ci/suppress_output python /ray/python/ray/rllib/tests/multiagent_pendulum.py docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh examples/multiagent_cartpole.py --num-iters=2 + /ray/ci/suppress_output python /ray/python/ray/rllib/examples/multiagent_cartpole.py --num-iters=2 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh examples/multiagent_two_trainers.py --num-iters=2 + /ray/ci/suppress_output python /ray/python/ray/rllib/examples/multiagent_two_trainers.py --num-iters=2 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh tests/test_avail_actions_qmix.py + /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_avail_actions_qmix.py docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh examples/cartpole_lstm.py --run=PPO --stop=200 + /ray/ci/suppress_output python /ray/python/ray/rllib/examples/cartpole_lstm.py --run=PPO --stop=200 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh examples/cartpole_lstm.py --run=IMPALA --stop=100 + /ray/ci/suppress_output python /ray/python/ray/rllib/examples/cartpole_lstm.py --run=IMPALA --stop=100 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh examples/cartpole_lstm.py --stop=200 --use-prev-action-reward - -# TODO(ericl): reenable the test after fix the arrow serialization error. -# https://github.com/ray-project/ray/pull/4127#issuecomment-468903577 -#docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ -# /ray/python/ray/rllib/tests/run_silent.sh examples/custom_loss.py --iters=2 + /ray/ci/suppress_output python /ray/python/ray/rllib/examples/cartpole_lstm.py --stop=200 --use-prev-action-reward docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh examples/custom_metrics_and_callbacks.py --num-iters=2 + /ray/ci/suppress_output python /ray/python/ray/rllib/examples/custom_loss.py --iters=2 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh contrib/random_agent/random_agent.py + /ray/ci/suppress_output python /ray/python/ray/rllib/examples/custom_metrics_and_callbacks.py --num-iters=2 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh examples/twostep_game.py --stop=2000 --run=PG + /ray/ci/suppress_output python /ray/python/ray/rllib/contrib/random_agent/random_agent.py docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh examples/twostep_game.py --stop=2000 --run=QMIX + /ray/ci/suppress_output python /ray/python/ray/rllib/examples/twostep_game.py --stop=2000 --run=PG docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh examples/twostep_game.py --stop=2000 --run=APEX_QMIX + /ray/ci/suppress_output python /ray/python/ray/rllib/examples/twostep_game.py --stop=2000 --run=QMIX docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output python /ray/python/ray/rllib/examples/twostep_game.py --stop=2000 --run=APEX_QMIX + +docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env PongDeterministic-v4 \ --run A3C \ --stop '{"training_iteration": 1}' \ --config '{"num_workers": 2, "use_pytorch": true, "sample_async": false, "model": {"use_lstm": false, "grayscale": true, "zero_mean": false, "dim": 84}, "preprocessor_pref": "rllib"}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env CartPole-v1 \ --run A3C \ --stop '{"training_iteration": 1}' \ --config '{"num_workers": 2, "use_pytorch": true, "sample_async": false}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh train.py \ + /ray/ci/suppress_output /ray/python/ray/rllib/train.py \ --env PongDeterministic-v4 \ --run IMPALA \ --stop='{"timesteps_total": 40000}' \ @@ -412,7 +408,7 @@ docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ --config '{"num_workers": 1, "num_gpus": 0, "num_envs_per_worker": 64, "sample_batch_size": 50, "train_batch_size": 50, "learner_queue_size": 1}' docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh agents/impala/vtrace_test.py + /ray/ci/suppress_output python /ray/python/ray/rllib/agents/impala/vtrace_test.py docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - /ray/python/ray/rllib/tests/run_silent.sh tests/test_ignore_worker_failure.py + /ray/ci/suppress_output python /ray/python/ray/rllib/tests/test_ignore_worker_failure.py diff --git a/ci/suppress_output b/ci/suppress_output index 18652d1ec..623559d11 100755 --- a/ci/suppress_output +++ b/ci/suppress_output @@ -2,7 +2,6 @@ # Run a command, suppressing output unless it hangs or crashes. TMPFILE=`mktemp` -COMMAND="$@" PID=$$ # Print output to avoid travis killing us @@ -20,7 +19,7 @@ watchdog() { watchdog & 2>/dev/null WATCHDOG_PID=$! -time $COMMAND >$TMPFILE 2>&1 +time "$@" >$TMPFILE 2>&1 CODE=$? if [ $CODE != 0 ]; then diff --git a/python/ray/rllib/examples/custom_loss.py b/python/ray/rllib/examples/custom_loss.py index 005428b00..4f15b9c96 100644 --- a/python/ray/rllib/examples/custom_loss.py +++ b/python/ray/rllib/examples/custom_loss.py @@ -51,9 +51,9 @@ class CustomLossModel(Model): input_ops = reader.tf_input_ops() # define a secondary loss by building a graph copy with weight sharing + obs = tf.cast(input_ops["obs"], tf.float32) logits, _ = self._build_layers_v2({ - "obs": restore_original_dimensions(input_ops["obs"], - self.obs_space) + "obs": restore_original_dimensions(obs, self.obs_space) }, self.num_outputs, self.options) # You can also add self-supervised losses easily by referencing tensors diff --git a/python/ray/rllib/offline/off_policy_estimator.py b/python/ray/rllib/offline/off_policy_estimator.py index dba85df7a..d09fe6baf 100644 --- a/python/ray/rllib/offline/off_policy_estimator.py +++ b/python/ray/rllib/offline/off_policy_estimator.py @@ -92,8 +92,9 @@ class OffPolicyEstimator(object): raise ValueError( "Off-policy estimation is not possible unless the inputs " "include action probabilities (i.e., the policy is stochastic " - "and emits the 'action_prob' key). You can set " - "`input_evaluation: []` to resolve this.") + "and emits the 'action_prob' key). For DQN this means using " + "`soft_q: True`. You can also set `input_evaluation: []` to " + "disable estimation.") @DeveloperAPI def get_metrics(self): diff --git a/python/ray/rllib/tests/data/cartpole_small/output-2019-02-03_20-27-20_worker-0_0.json b/python/ray/rllib/tests/data/cartpole_small/output-2019-02-03_20-27-20_worker-0_0.json index 0030cb2d7..803617e91 100644 --- a/python/ray/rllib/tests/data/cartpole_small/output-2019-02-03_20-27-20_worker-0_0.json +++ b/python/ray/rllib/tests/data/cartpole_small/output-2019-02-03_20-27-20_worker-0_0.json @@ -1,4 +1,3 @@ -{"advantages": [19.83694076538086, 19.02721405029297, 18.209306716918945, 17.383136749267578, 16.54862403869629, 15.705680847167969, 14.854223251342773, 13.99416446685791, 13.125418663024902, 12.24789810180664, 11.361513137817383, 10.466174125671387, 9.561792373657227, 8.648275375366211, 7.725530624389648, 6.7934651374816895, 5.851984977722168, 4.900994777679443, 3.940398931503296, 2.970099925994873, 1.9900000095367432, 1.0], "eps_id": [767029556, 767029556, 767029556, 767029556, 767029556, 767029556, 767029556, 767029556, 767029556, 767029556, 767029556, 767029556, 767029556, 767029556, 767029556, 767029556, 767029556, 767029556, 767029556, 767029556, 767029556, 767029556], "action_prob": [0.49987873435020447, 0.4997735917568207, 0.5001150369644165, 0.5004363656044006, 0.5005561709403992, 0.5005311965942383, 0.4995429217815399, 0.499515563249588, 0.49954167008399963, 0.49970272183418274, 0.5000479817390442, 0.4995552599430084, 0.5001161694526672, 0.5005202293395996, 0.4992202818393707, 0.49941039085388184, 0.4997551143169403, 0.49998918175697327, 0.5000606179237366, 0.5000531673431396, 0.49997803568840027, 0.49995389580726624], "dones": [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, true], "type": "SampleBatch", "value_targets": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "agent_index": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "obs": "BCJNGGhAgAQAAAAAAAAg5QIAAFAAAAAAAQUAAwIAUcQAAAAQDACwCgAMAAYABQAIAAoQAOMBAwAMAAAACAAIAAAABAYAAjwAAAgA0cj///8AAAEOeAAAABgiAAIcABEwNAAASgACOgAAFgAEHAAAAgCREAAUAAgABgAHXgATEHwAcwECJAAAABRIAAACAABEADEIAAd4AACzAPEAIAAAAAcAAABuZGFycmF5VABBbGlzdB4AQwAAAMxAAAACADMMABbKABEMZgAxAAMDpAAAqAACAgBRCgAYAAzWAADsABNsBAEIGAESBSkADwIAAgASAQACAAwIAAJIAQYCAAQQAAQoAAACABACzQAACQAHAgAIeAAPAgBNGfwoARoaKAEwBAMAaAEZYJEAMQ4AKIgBEQy8ARIOoQBjA4wAAAA0/AEEAgAIOAAA6AAEAAEArAEEAgAAGAAAJAICXAIxEAAIhgECEAAIKAAAAgARCCwCAnwCAEwAEBYVAAkCAABmAjEGAAYUAAm1AA8CAAb//1I/TQc9XIHsPAL2WrxBCaI8s6oJPfoKKr7CelS8ic6dPnIg+DwJl+88ZPfeu0r0Ozwl6/w8VtplPr5y17tHE5G+9tgQPWfg1j6Ckki8Vf4Tv046Mz3Rbh0/ggDDvK/mX780m2U9/3ZPPzsmKb0PXJa/Xf+TPTSoHT+TsIS9ri1lv/84rT0B5Nc+uVupveejH79vfr49l0BpPpbmwr2EU7e+8NLHPVzSCz0ckdG9CVLGveI4yT3zNCO+gojVvZbGJT6lscI989IXPffmzr2VqSW+UDbEPf46IL5Zh9W9I3rHPY7NvT36LbO+B4rRvTP+tT71d689iTodvs/6wr0negQ97y2pPeRULz2rp8G97dyTvsnuqj0Q4XQ+53vNveAXHL9ZurQ9PfvePnx15r3GqG6/AZHGPePHIT98Uga+zwihv4xz4D3rF1Q/dBYgvu1dy79yMQE+Q7IiP1ygQL5Uoay/egEHYAAAAAAAAAAAAAA=", "prev_rewards": [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "infos": [{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}], "rewards": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "new_obs": "BCJNGGhAgAQAAAAAAAAg5QIAAFAAAAAAAQUAAwIAUcQAAAAQDACwCgAMAAYABQAIAAoQAOMBAwAMAAAACAAIAAAABAYAAjwAAAgA0cj///8AAAEOeAAAABgiAAIcABEwNAAASgACOgAAFgAEHAAAAgCREAAUAAgABgAHXgATEHwAcwECJAAAABRIAAACAABEADEIAAd4AACzAPEAIAAAAAcAAABuZGFycmF5VABBbGlzdB4AQwAAAMxAAAACADMMABbKABEMZgAxAAMDpAAAqAACAgBRCgAYAAzWAADsABNsBAEIGAESBSkADwIAAgASAQACAAwIAAJIAQYCAAQQAAQoAAACABACzQAACQAHAgAIeAAPAgBNGfwoARoaKAEwBAMAaAEZYJEAMQ4AKIgBEQy8ARIOoQBjA4wAAAA0/AEEAgAIOAAA6AAEAAEArAEEAgAAGAAAJAICXAIxEAAIhgECEAAIKAAAAgARCCwCAnwCAEwAEBYVAAkCAABmAjEGAAYUAAm1AA8CAAb//1Kzqgk9+goqvsJ6VLyJzp0+ciD4PAmX7zxk9967SvQ7PCXr/DxW2mU+vnLXu0cTkb722BA9Z+DWPoKSSLxV/hO/TjozPdFuHT+CAMO8r+ZfvzSbZT3/dk8/OyYpvQ9clr9d/5M9NKgdP5OwhL2uLWW//zitPQHk1z65W6m956Mfv29+vj2XQGk+lubCvYRTt77w0sc9XNILPRyR0b0JUsa94jjJPfM0I76CiNW9lsYlPqWxwj3z0hc99+bOvZWpJb5QNsQ9/jogvlmH1b0jesc9js29Pfots74HitG9M/61PvV3rz2JOh2+z/rCvSd6BD3vLak95FQvPaunwb3t3JO+ye6qPRDhdD7ne8294Bccv1m6tD09+94+fHXmvcaobr8BkcY948chP3xSBr7PCKG/jHPgPesXVD90FiC+7V3Lv3IxAT5DsiI/XKBAvlShrL94NQ4+qhtVP0s/XL4xz9i/egEHYAAAAAAAAAAAAAA=", "prev_actions": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "actions": [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1], "t": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]} -{"advantages": [13.125418663024902, 12.24789810180664, 11.361513137817383, 10.466174125671387, 9.561792373657227, 8.648275375366211, 7.725530624389648, 6.7934651374816895, 5.851984977722168, 4.900994777679443, 3.940398931503296, 2.970099925994873, 1.9900000095367432, 1.0], "eps_id": [312569639, 312569639, 312569639, 312569639, 312569639, 312569639, 312569639, 312569639, 312569639, 312569639, 312569639, 312569639, 312569639, 312569639], "action_prob": [0.4998626112937927, 0.5002043843269348, 0.49963468313217163, 0.49981942772865295, 0.4998198449611664, 0.5001484155654907, 0.5002940893173218, 0.4997011423110962, 0.5002346634864807, 0.5002323985099792, 0.4998157322406769, 0.5001403093338013, 0.49990716576576233, 0.49997448921203613], "dones": [false, false, false, false, false, false, false, false, false, false, false, false, false, true], "type": "SampleBatch", "value_targets": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "agent_index": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "obs": "BCJNGGhAAAQAAAAAAABwYgIAAFAAAAAAAQUAAwIAUcQAAAAQDACwCgAMAAYABQAIAAoQAOMBAwAMAAAACAAIAAAABAYAAjwAAAgA0cj///8AAAEOeAAAABgiAAIcABEwNAAASgACOgAAFgAEHAAAAgCREAAUAAgABgAHXgATEHwAcwECJAAAABRIAAACAABEADEIAAd4AACzAPEAIAAAAAcAAABuZGFycmF5VABBbGlzdB4AQwAAAMxAAAACADMMABbKABEMZgAxAAMDpAAAqAACAgBRCgAYAAzWAADsABNsBAEIGAESBSkADwIAAgASAQACAAwIAAJIAQYCAAQQAAQoAAACABACzQAACQAHAgAIeAAPAgBNGfwoARoaKAEwBAMAaAEZ4IUAMQ4AKIgBEQy8ARIOHABjA4wAAAA0/AEEAgAIOAAA6AAEAAEArAEEAgAAGAAAJAICXAIxEAAIhgECEAAIKAAAAgARCCwCAnwCAEwAA3wABwIAAGYCMQYABhIAD0YBE/HR9EURvWhK+jyOCpA8wk1+OzbFDr2uxSi+T62QPHGnmj6qRRy9RGq4vpIqwjyPqhk/Tsc5vXlfKb6gQRI9WVmhPhBUR73MJO88exIsPfbyCj3b70S9kYAqvubZLj2t2q0+vpNSvWhzub74qko9aDglP84/cL0I2A6/0Yl/PU4HdD/K+o69AT66vlHQpj0tSC4/B+GdvRhWD7/ossI9iXp/PxXQtL3lk0G/UJPrPYzSqD8HydO9AQsQv5/MED5uPIg/B9XqvbZkQr/amCY+9eyyP7H3BL5yCxG/pTlDPpF/lD8LAQ8CAANQAAAAAAAAAAAA", "prev_rewards": [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "infos": [{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}], "rewards": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "new_obs": "BCJNGGhAAAQAAAAAAABwYgIAAFAAAAAAAQUAAwIAUcQAAAAQDACwCgAMAAYABQAIAAoQAOMBAwAMAAAACAAIAAAABAYAAjwAAAgA0cj///8AAAEOeAAAABgiAAIcABEwNAAASgACOgAAFgAEHAAAAgCREAAUAAgABgAHXgATEHwAcwECJAAAABRIAAACAABEADEIAAd4AACzAPEAIAAAAAcAAABuZGFycmF5VABBbGlzdB4AQwAAAMxAAAACADMMABbKABEMZgAxAAMDpAAAqAACAgBRCgAYAAzWAADsABNsBAEIGAESBSkADwIAAgASAQACAAwIAAJIAQYCAAQQAAQoAAACABACzQAACQAHAgAIeAAPAgBNGfwoARoaKAEwBAMAaAEZ4IUAMQ4AKIgBEQy8ARIOHABjA4wAAAA0/AEEAgAIOAAA6AAEAAEArAEEAgAAGAAAJAICXAIxEAAIhgECEAAIKAAAAgARCCwCAnwCAEwAA3wABwIAAGYCMQYABhIAD0YBE/HRNsUOva7FKL5PrZA8caeaPqpFHL1Eari+kirCPI+qGT9Oxzm9eV8pvqBBEj1ZWaE+EFRHvcwk7zx7Eiw99vIKPdvvRL2RgCq+5tkuPa3arT6+k1K9aHO5vviqSj1oOCU/zj9wvQjYDr/RiX89Tgd0P8r6jr0BPrq+UdCmPS1ILj8H4Z29GFYPv+iywj2Jen8/FdC0veWTQb9Qk+s9jNKoPwfJ070BCxC/n8wQPm48iD8H1eq9tmRCv9qYJj717LI/sfcEvnILEb+lOUM+kX+UPzWSEL4yr7++I/xaPtbKbj8LAQ8CAANQAAAAAAAAAAAA", "prev_actions": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "actions": [0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1], "t": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]} -{"advantages": [41.296321868896484, 40.70335388183594, 40.104400634765625, 39.499393463134766, 38.888275146484375, 38.2709846496582, 37.6474609375, 37.017635345458984, 36.38145065307617, 35.73883819580078, 35.08973693847656, 34.434078216552734, 33.77179718017578, 33.10282516479492, 32.427093505859375, 31.74454116821289, 31.055091857910156, 30.358678817749023, 29.655229568481445, 28.944677352905273, 28.226945877075195, 27.50196647644043, 26.769662857055664, 26.02996253967285, 25.282791137695312, 24.5280704498291, 23.765727996826172, 22.995685577392578, 22.217864990234375, 21.432186126708984, 20.63857078552246, 19.83694076538086, 19.02721405029297, 18.209306716918945, 17.383136749267578, 16.54862403869629, 15.705680847167969, 14.854223251342773, 13.99416446685791, 13.125418663024902, 12.24789810180664, 11.361513137817383, 10.466174125671387, 9.561792373657227, 8.648275375366211, 7.725530624389648, 6.7934651374816895, 5.851984977722168, 4.900994777679443, 3.940398931503296, 2.970099925994873, 1.9900000095367432, 1.0], "eps_id": [1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029, 1121492029], "action_prob": [0.4999432861804962, 0.49970677495002747, 0.49994567036628723, 0.49970558285713196, 0.4999447166919708, 0.4997081458568573, 0.49994054436683655, 0.4997144043445587, 0.5000669360160828, 0.49960795044898987, 0.500072181224823, 0.4996034502983093, 0.4999268054962158, 0.49972784519195557, 0.4999243915081024, 0.4997323453426361, 0.5000817775726318, 0.500409722328186, 0.49946051836013794, 0.4995920956134796, 0.5000770688056946, 0.4996024966239929, 0.5000657439231873, 0.49961769580841064, 0.5000503659248352, 0.49963781237602234, 0.500030517578125, 0.5003365874290466, 0.5004424452781677, 0.49958422780036926, 0.49961057305336, 0.4997643828392029, 0.5001019239425659, 0.5004988312721252, 0.500738799571991, 0.5007926821708679, 0.4992518424987793, 0.5008513927459717, 0.5007851719856262, 0.4993096590042114, 0.4992087185382843, 0.5008699297904968, 0.5007670521736145, 0.4993574619293213, 0.4992888867855072, 0.5007497668266296, 0.500627875328064, 0.49950355291366577, 0.49948444962501526, 0.499498575925827, 0.49960172176361084, 0.5001403093338013, 0.49976104497909546], "dones": [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, true], "type": "SampleBatch", "value_targets": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "agent_index": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "obs": "BCJNGGhAgAYAAAAAAACa2wQAAFAAAAAAAQUAAwIAUcQAAAAQDACwCgAMAAYABQAIAAoQAOMBAwAMAAAACAAIAAAABAYAAjwAAAgA0cj///8AAAEOeAAAABgiAAIcABEwNAAASgACOgAAFgAEHAAAAgCREAAUAAgABgAHXgATEHwAcwECJAAAABRIAAACAABEADEIAAd4AACzAPEAIAAAAAcAAABuZGFycmF5VABBbGlzdB4AQwAAAMxAAAACADMMABbKABEMZgAxAAMDpAAAqAACAgBRCgAYAAzWAADsABNsBAEIGAESBSkADwIAAgASAQACAAwIAAJIAQYCAAQQAAQoAAACABACzQAACQAHAgAIeAAPAgBNGfwoARoaKAEwBAMAaAEoUAOGADEOACiIAREMvAESDhsAYwOMAAAANPwBBAIACDgAAOgABAABAKwBBAIAABgAACQCAlwCMRAACIYBAhAACCgAAAIAEQgsAgJ8AgBMABA1FQAJAgAAZgIxBgAGFAAPRgET8f///0h0hq67dp2wPBLe3Ltce6i8YGWgu9ugMb6UWOq7FjuKPgAKCbwGPrI8caTluqlfzLzN6AG8s38xvl6FE7sJzIg+gbU6vAe2sjztrUo7K7fWvIGPM7zjgjG+LlMoO8buiD46XWy8PAyyPNq3ATz8Fsi8BD5lvAyqMb7cbfM7I5+KPiMMj7wMQLA8s25SPDNxoLy8hYu8KKVdPsUDTDzbyp2+TR5QvHdJrTwgDs47pCA+vNcvSbxyX10+OXPGO6PGmr77WAK8P9erPCd9LDcCpPy7qfL2uy5TMr7S6Ba5ZuiRPrWJNLw+26s8zAu2O4gF/rvmqS28amcyvjT3sDuFx5I+wMBmvLCSqjzwazY8zseMuxXuX7wR9lw+igM1PH8+lr7xOBm8L03SPhS3qTvTJhW/JgmVuma5XD7zHNS72p2TvlYCUDujHag8+4dIvBUWGDtb6Go7NP9cPk/FR7w2oZa+J3IBPDP6qjw+FpS8w32wu/ZICDwMcF0+J/iUvNSAm74jJU88PDmvPPq6xrxX04m8bSdWPCoOXj6lfMm85VWivhmbjjxe8bQ8K2/9vHIJBL2HOZI829xePp1bAb3DRau+/OG1PHWO0z7wwhy9txEjv7CU+TxQ3Bs/kPFQvXsEcb9yqi49YzDUPtwIj72cTSq/tZ1QPa2iYT56SKq90mrKvriqYj3WENk8+3m6vdQcBb5p1mQ9MTkrvg7Nv73M0Ac+wSNXPZLEuL5OXrq92enJPqqTOT35+Q2/HjeqvbkyKD/0JAw9nJs/v7hNj73aI2w/36idPPJuDb/ZClO95swbP9tIBjzdMD+/ri8hvR52Yj8j4dy79P9wv+1vsbz2E5U/sXXRvGj6Pr8TH9Y6zoJdP8fXJb1aCA2/TCabPDq3Ej8r+VK97gw/vzQM+TyJJ18/Aw6Ivacacb/q7kM9dT6WP6Ohrr30UT+/cQuSPQx/ZT8cPs29TaINv5zDtj3ihCA/cOfjvYffP795ctA9H0lyP0hNAb6BInK/fjb3PbJ3oj80rBS+ZKBAv+uZFT5YI4I/MRUkvjo+D79fbCo+WvJFP9CKL76H7ru+UkI6PkWFCT87Dze+Hhczvr5CRT4QIJ0+LaQ6viiMvr60i0s+hFInPwAAAAB/Aw8CAA9QAAAAAAAAAAAA", "prev_rewards": [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "infos": [{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}], "rewards": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "new_obs": "BCJNGGhAgAYAAAAAAACa2wQAAFAAAAAAAQUAAwIAUcQAAAAQDACwCgAMAAYABQAIAAoQAOMBAwAMAAAACAAIAAAABAYAAjwAAAgA0cj///8AAAEOeAAAABgiAAIcABEwNAAASgACOgAAFgAEHAAAAgCREAAUAAgABgAHXgATEHwAcwECJAAAABRIAAACAABEADEIAAd4AACzAPEAIAAAAAcAAABuZGFycmF5VABBbGlzdB4AQwAAAMxAAAACADMMABbKABEMZgAxAAMDpAAAqAACAgBRCgAYAAzWAADsABNsBAEIGAESBSkADwIAAgASAQACAAwIAAJIAQYCAAQQAAQoAAACABACzQAACQAHAgAIeAAPAgBNGfwoARoaKAEwBAMAaAEoUAOGADEOACiIAREMvAESDhsAYwOMAAAANPwBBAIACDgAAOgABAABAKwBBAIAABgAACQCAlwCMRAACIYBAhAACCgAAAIAEQgsAgJ8AgBMABA1FQAJAgAAZgIxBgAGFAAPRgET8f///0hgZaC726AxvpRY6rsWO4o+AAoJvAY+sjxxpOW6qV/MvM3oAbyzfzG+XoUTuwnMiD6BtTq8B7ayPO2tSjsrt9a8gY8zvOOCMb4uUyg7xu6IPjpdbLw8DLI82rcBPPwWyLwEPmW8DKoxvtxt8zsjn4o+IwyPvAxAsDyzblI8M3GgvLyFi7wopV0+xQNMPNvKnb5NHlC8d0mtPCAOzjukID681y9JvHJfXT45c8Y7o8aavvtYArw/16s8J30sNwKk/Lup8va7LlMyvtLoFrlm6JE+tYk0vD7bqzzMC7Y7iAX+u+apLbxqZzK+NPewO4XHkj7AwGa8sJKqPPBrNjzOx4y7Fe5fvBH2XD6KAzU8fz6WvvE4GbwvTdI+FLepO9MmFb8mCZW6ZrlcPvMc1LvanZO+VgJQO6MdqDz7h0i8FRYYO1voajs0/1w+T8VHvDahlr4ncgE8M/qqPD4WlLzDfbC79kgIPAxwXT4n+JS81ICbviMlTzw8Oa88+rrGvFfTibxtJ1Y8Kg5ePqV8ybzlVaK+GZuOPF7xtDwrb/28cgkEvYc5kjzb3F4+nVsBvcNFq7784bU8dY7TPvDCHL23ESO/sJT5PFDcGz+Q8VC9ewRxv3KqLj1jMNQ+3AiPvZxNKr+1nVA9raJhPnpIqr3Sasq+uKpiPdYQ2Tz7ebq91BwFvmnWZD0xOSu+Ds2/vczQBz7BI1c9ksS4vk5eur3Z6ck+qpM5Pfn5Db8eN6q9uTIoP/QkDD2cmz+/uE2PvdojbD/fqJ088m4Nv9kKU73mzBs/20gGPN0wP7+uLyG9HnZiPyPh3Lv0/3C/7W+xvPYTlT+xddG8aPo+vxMf1jrOgl0/x9clvVoIDb9MJps8OrcSPyv5Ur3uDD+/NAz5PIknXz8DDoi9pxpxv+ruQz11PpY/o6GuvfRRP79xC5I9DH9lPxw+zb1Nog2/nMO2PeKEID9w5+O9h98/v3ly0D0fSXI/SE0BvoEicr9+Nvc9sneiPzSsFL5koEC/65kVPlgjgj8xFSS+Oj4Pv19sKj5a8kU/0Iovvofuu75SQjo+RYUJPzsPN74eFzO+vkJFPhAgnT4tpDq+KIy+vrSLSz6EUic/YUNCvvmbOL537lg+HOjbPgAAAAB/Aw8CAA9QAAAAAAAAAAAA", "prev_actions": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "actions": [0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1], "t": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52]} -{"advantages": [11.361513137817383, 10.466174125671387, 9.561792373657227, 8.648275375366211, 7.725530624389648, 6.7934651374816895, 5.851984977722168, 4.900994777679443, 3.940398931503296, 2.970099925994873, 1.9900000095367432, 1.0], "eps_id": [701622330, 701622330, 701622330, 701622330, 701622330, 701622330, 701622330, 701622330, 701622330, 701622330, 701622330, 701622330], "action_prob": [0.4999730587005615, 0.5003005862236023, 0.5004333257675171, 0.49958109855651855, 0.49961304664611816, 0.5002139210700989, 0.5003206729888916, 0.4997044503688812, 0.5002343058586121, 0.5002065300941467, 0.5001469850540161, 0.49990975856781006], "dones": [false, false, false, false, false, false, false, false, false, false, false, true], "type": "SampleBatch", "value_targets": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "agent_index": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "obs": "BCJNGGhAwAMAAAAAAAArNgIAAFAAAAAAAQUAAwIAUcQAAAAQDACwCgAMAAYABQAIAAoQAOMBAwAMAAAACAAIAAAABAYAAjwAAAgA0cj///8AAAEOeAAAABgiAAIcABEwNAAASgACOgAAFgAEHAAAAgCREAAUAAgABgAHXgATEHwAcwECJAAAABRIAAACAABEADEIAAd4AACzAPEAIAAAAAcAAABuZGFycmF5VABBbGlzdB4AQwAAAMxAAAACADMMABbKABEMZgAxAAMDpAAAqAACAgBRCgAYAAzWAADsABNsBAEIGAESBSkADwIAAgASAQACAAwIAAJIAQYCAAQQAAQoAAACABACzQAACQAHAgAIeAAPAgBNGfwoARoaKAEwBAMAaAEZwIUAMQ4AKIgBEQy8ARIOHABjA4wAAAA0/AEEAgAIOAAA6AAEAAEArAEEAgAAGAAAJAICXAIxEAAIhgECEAAIKAAAAgARCCwCAnwCAEwAADQACgIAAGYCMQYABhUAD0YBE/CxMoQDvRjiELxH6ta8JzhvO6U9BL3sHz8+LlHWvLgpmL7T5um8kKbDPi+BA71AFxm/IEurvEDjEz9jfjS9yY9mv2lKGbxAMcQ+/0V+vS5DH78D0t26boVBPmOemL3GALO+B8wIO7cpxT5e8Ka9pTIqvy9iIDwbzBQ/rCvCvVBme78KbK88UmfGPgBl6r0ydzi/Q+nuPLmBFT9b9AO+JQOGvzlMJz1x1Ec/gGUZvhFHsL9JPmc9jit6P9WZNb7rOdu/AAAAAA==", "prev_rewards": [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "infos": [{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}], "rewards": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "new_obs": "BCJNGGhAwAMAAAAAAAArNgIAAFAAAAAAAQUAAwIAUcQAAAAQDACwCgAMAAYABQAIAAoQAOMBAwAMAAAACAAIAAAABAYAAjwAAAgA0cj///8AAAEOeAAAABgiAAIcABEwNAAASgACOgAAFgAEHAAAAgCREAAUAAgABgAHXgATEHwAcwECJAAAABRIAAACAABEADEIAAd4AACzAPEAIAAAAAcAAABuZGFycmF5VABBbGlzdB4AQwAAAMxAAAACADMMABbKABEMZgAxAAMDpAAAqAACAgBRCgAYAAzWAADsABNsBAEIGAESBSkADwIAAgASAQACAAwIAAJIAQYCAAQQAAQoAAACABACzQAACQAHAgAIeAAPAgBNGfwoARoaKAEwBAMAaAEZwIUAMQ4AKIgBEQy8ARIOHABjA4wAAAA0/AEEAgAIOAAA6AAEAAEArAEEAgAAGAAAJAICXAIxEAAIhgECEAAIKAAAAgARCCwCAnwCAEwAADQACgIAAGYCMQYABhUAD0YBE/CxpT0EvewfPz4uUda8uCmYvtPm6byQpsM+L4EDvUAXGb8gS6u8QOMTP2N+NL3Jj2a/aUoZvEAxxD7/RX69LkMfvwPS3bpuhUE+Y56YvcYAs74HzAg7tynFPl7wpr2lMiq/L2IgPBvMFD+sK8K9UGZ7vwpsrzxSZ8Y+AGXqvTJ3OL9D6e48uYEVP1v0A74lA4a/OUwnPXHURz+AZRm+EUewv0k+Zz2OK3o/1Zk1vus5278cpps97NZIP1atWL6Pcr2/AAAAAA==", "prev_actions": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "actions": [1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0], "t": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]} +{"type": "SampleBatch", "weights": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "eps_id": [241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760, 241561760], "dones": [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, true], "infos": [{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}], "prev_rewards": [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "t": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55], "agent_index": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "action_prob": [0.4979577958583832, 0.5745141506195068, 0.5042742490768433, 0.5248998403549194, 0.5048907995223999, 0.5254997611045837, 0.4930223524570465, 0.5723332166671753, 0.5071576237678528, 0.5262983441352844, 0.5075111389160156, 0.4721700847148895, 0.4541035294532776, 0.5691784024238586, 0.45002007484436035, 0.42802754044532776, 0.5951988697052002, 0.5743389129638672, 0.44297751784324646, 0.5751434564590454, 0.4427056908607483, 0.575354278087616, 0.5583169460296631, 0.5349109768867493, 0.49323225021362305, 0.42819857597351074, 0.6240300536155701, 0.42723774909973145, 0.6247843503952026, 0.4268564283847809, 0.6255699396133423, 0.5718400478363037, 0.49357253313064575, 0.5718478560447693, 0.506999135017395, 0.4627947509288788, 0.44369709491729736, 0.42281273007392883, 0.40176495909690857, 0.6177492141723633, 0.6000679731369019, 0.4211883246898651, 0.5995147228240967, 0.578464925289154, 0.5586039423942566, 0.5260810256004333, 0.4879906177520752, 0.42811155319213867, 0.6308852434158325, 0.5760338306427002, 0.5073276162147522, 0.46694710850715637, 0.43938523530960083, 0.5832104086875916, 0.5628215670585632, 0.5309032201766968], "actions": [0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0], "q_values": [[-0.005643954500555992, 0.0025248583406209946], [-0.04723002016544342, 0.2530632019042969], [-0.004162287805229425, 0.012935103848576546], [0.05779631435871124, -0.041885510087013245], [-0.0001599406823515892, 0.019403917714953423], [0.05187809467315674, -0.05020952224731445], [-6.351247429847717e-05, 0.027848877012729645], [-0.03533334285020828, 0.2560437023639679], [0.005023432895541191, 0.03365574777126312], [0.04525064304471016, -0.06003996357321739], [0.002838471904397011, 0.032885171473026276], [0.03723599761724472, -0.07419878989458084], [0.09575563669204712, -0.0883483961224556], [0.16416001319885254, -0.11433979868888855], [0.09313704073429108, -0.10745253413915634], [0.16196757555007935, -0.12793570756912231], [0.23910409212112427, -0.1463954746723175], [0.15805242955684662, -0.14152376353740692], [0.09662380814552307, -0.1324627697467804], [0.1541520208120346, -0.14871598780155182], [0.0929112657904625, -0.1372770369052887], [0.1511463224887848, -0.15258446335792542], [0.0875367745757103, -0.14679750800132751], [0.08854943513870239, -0.05132210999727249], [0.018426118418574333, 0.045498818159103394], [-0.04996141046285629, 0.23924344778060913], [-0.09354546666145325, 0.4131438434123993], [-0.038044273853302, 0.255085825920105], [-0.09211604297161102, 0.4177895784378052], [-0.030748017132282257, 0.26394063234329224], [-0.09104493260383606, 0.4222134053707123], [-0.02319370210170746, 0.2661687135696411], [0.02133956551551819, 0.04705086350440979], [-0.021654099225997925, 0.2677402198314667], [0.01794305630028248, 0.04594135284423828], [0.05681019276380539, -0.0922863557934761], [0.11023147404193878, -0.1159394159913063], [0.16652457416057587, -0.14471273124217987], [0.23569053411483765, -0.16242587566375732], [0.31461724638938904, -0.165388286113739], [0.22523169219493866, -0.1805165857076645], [0.14499591290950775, -0.17290116846561432], [0.2126035839319229, -0.19084002077579498], [0.12525871396064758, -0.19121608138084412], [0.07890036702156067, -0.15659788250923157], [0.07070913910865784, -0.03370969370007515], [-0.0010413788259029388, 0.047005534172058105], [-0.05502410978078842, 0.2345360815525055], [-0.15737640857696533, 0.37863999605178833], [-0.09506852179765701, 0.21144413948059082], [-0.06340484321117401, -0.0340922586619854], [0.016717009246349335, -0.11568755656480789], [0.059842679649591446, -0.1838146150112152], [0.12809047102928162, -0.20787617564201355], [0.055311597883701324, -0.19730976223945618], [-0.022230863571166992, -0.14600159227848053]], "rewards": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "prev_actions": [0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0], "obs": [[0.040251147001981735, -0.009447001852095127, 0.04735473543405533, -0.00123753328807652], [0.040062207728624344, -0.2052149772644043, 0.04732998460531235, 0.30600231885910034], [0.03595791012048721, -0.010798314586281776, 0.05345002934336662, 0.028613731265068054], [0.03574194014072418, 0.18351800739765167, 0.054022304713726044, -0.24673765897750854], [0.039412301033735275, -0.012332209385931492, 0.04908755049109459, 0.06248391792178154], [0.03916565701365471, 0.1820528209209442, 0.050337228924036026, -0.2143164724111557], [0.0428067147731781, -0.013751287944614887, 0.04605090245604515, 0.09381057322025299], [0.04253168776631355, -0.20950199663639069, 0.04792711138725281, 0.4006595313549042], [0.03834164887666702, -0.015091483481228352, 0.055940303951501846, 0.12346379458904266], [0.03803981840610504, 0.17918626964092255, 0.05840957909822464, -0.1510591059923172], [0.041623543947935104, -0.01672130823135376, 0.055388398468494415, 0.1594637781381607], [0.04128911718726158, 0.17756572365760803, 0.05857767164707184, -0.11524398624897003], [0.044840432703495026, 0.37180155515670776, 0.05627279356122017, -0.3888860046863556], [0.05227646231651306, 0.5660815238952637, 0.04849507287144661, -0.6633091568946838], [0.06359809637069702, 0.3703196048736572, 0.03522888943552971, -0.3557596206665039], [0.0710044875741005, 0.5649234652519226, 0.028113696724176407, -0.6371290683746338], [0.08230295777320862, 0.7596423029899597, 0.015371114946901798, -0.9208275675773621], [0.09749580174684525, 0.5643160343170166, -0.003045437391847372, -0.623353898525238], [0.10878212004899979, 0.36923670768737793, -0.015512514859437943, -0.3316316306591034], [0.116166852414608, 0.5645759701728821, -0.022145148366689682, -0.6291658282279968], [0.1274583786725998, 0.36976999044418335, -0.03472846373915672, -0.3435385525226593], [0.1348537802696228, 0.5653683543205261, -0.04159923642873764, -0.6469672918319702], [0.14616113901138306, 0.3708499073982239, -0.054538581520318985, -0.3676687479019165], [0.15357813239097595, 0.17654363811016083, -0.06189195439219475, -0.09266908466815948], [0.15710900723934174, -0.01763911545276642, -0.06374533474445343, 0.17986272275447845], [0.1567562371492386, -0.2117937058210373, -0.06014808267354965, 0.4517746567726135], [0.15252035856246948, -0.4060157239437103, -0.0511125884950161, 0.7249079942703247], [0.14440004527568817, -0.21022562682628632, -0.03661442920565605, 0.4165858030319214], [0.14019553363323212, -0.4048100411891937, -0.028282713145017624, 0.6975045800209045], [0.13209933042526245, -0.20930756628513336, -0.014332621358335018, 0.39605414867401123], [0.12791317701339722, -0.4042232632637024, -0.006411538925021887, 0.6841840147972107], [0.1198287084698677, -0.20901288092136383, 0.007272141519933939, 0.38948947191238403], [0.11564845591783524, -0.013994891196489334, 0.015061930753290653, 0.09910821169614792], [0.11536855250597, -0.20932942628860474, 0.01704409532248974, 0.39650481939315796], [0.11118196696043015, -0.014453399926424026, 0.024974191561341286, 0.1092439591884613], [0.1108928993344307, 0.18030193448066711, 0.0271590705960989, -0.17545630037784576], [0.11449893563985825, 0.3750248849391937, 0.023649943992495537, -0.45944923162460327], [0.12199943512678146, 0.5698046684265137, 0.014460960403084755, -0.7445847988128662], [0.13339552283287048, 0.7647241353988647, -0.000430735235568136, -1.032681941986084], [0.14869001507759094, 0.9598518013954163, -0.02108437567949295, -1.3255001306533813], [0.16788704693317413, 0.7650023102760315, -0.047594375908374786, -1.0394892692565918], [0.1831870973110199, 0.5705440044403076, -0.06838416308164597, -0.762119472026825], [0.1945979744195938, 0.7665379047393799, -0.08362655341625214, -1.0755125284194946], [0.2099287360906601, 0.5726144313812256, -0.10513680428266525, -0.8102014064788818], [0.2213810235261917, 0.3790779709815979, -0.12134082615375519, -0.552353024482727], [0.22896258533000946, 0.1858503371477127, -0.1323878914117813, -0.30022940039634705], [0.2326795905828476, -0.007160619366914034, -0.13839247822761536, -0.05205482989549637], [0.23253637552261353, -0.2000548243522644, -0.1394335776567459, 0.1939624696969986], [0.22853527963161469, -0.3929353952407837, -0.13555432856082916, 0.4396146833896637], [0.22067657113075256, -0.1961815357208252, -0.1267620325088501, 0.10746019333600998], [0.21675294637680054, 0.0005075104418210685, -0.12461283057928085, -0.22237446904182434], [0.21676309406757355, 0.19716985523700714, -0.1290603131055832, -0.5516219735145569], [0.2207064926624298, 0.39384564757347107, -0.14009276032447815, -0.8820206522941589], [0.22858339548110962, 0.5905638933181763, -0.15773317217826843, -1.2152597904205322], [0.2403946816921234, 0.39778846502304077, -0.18203836679458618, -0.9758678674697876], [0.24835044145584106, 0.20551282167434692, -0.20155572891235352, -0.745444118976593]], "new_obs": [[0.040062207728624344, -0.2052149772644043, 0.04732998460531235, 0.30600231885910034], [0.03595791012048721, -0.010798314586281776, 0.05345002934336662, 0.028613731265068054], [0.03574194014072418, 0.18351800739765167, 0.054022304713726044, -0.24673765897750854], [0.039412301033735275, -0.012332209385931492, 0.04908755049109459, 0.06248391792178154], [0.03916565701365471, 0.1820528209209442, 0.050337228924036026, -0.2143164724111557], [0.0428067147731781, -0.013751287944614887, 0.04605090245604515, 0.09381057322025299], [0.04253168776631355, -0.20950199663639069, 0.04792711138725281, 0.4006595313549042], [0.03834164887666702, -0.015091483481228352, 0.055940303951501846, 0.12346379458904266], [0.03803981840610504, 0.17918626964092255, 0.05840957909822464, -0.1510591059923172], [0.041623543947935104, -0.01672130823135376, 0.055388398468494415, 0.1594637781381607], [0.04128911718726158, 0.17756572365760803, 0.05857767164707184, -0.11524398624897003], [0.044840432703495026, 0.37180155515670776, 0.05627279356122017, -0.3888860046863556], [0.05227646231651306, 0.5660815238952637, 0.04849507287144661, -0.6633091568946838], [0.06359809637069702, 0.3703196048736572, 0.03522888943552971, -0.3557596206665039], [0.0710044875741005, 0.5649234652519226, 0.028113696724176407, -0.6371290683746338], [0.08230295777320862, 0.7596423029899597, 0.015371114946901798, -0.9208275675773621], [0.09749580174684525, 0.5643160343170166, -0.003045437391847372, -0.623353898525238], [0.10878212004899979, 0.36923670768737793, -0.015512514859437943, -0.3316316306591034], [0.116166852414608, 0.5645759701728821, -0.022145148366689682, -0.6291658282279968], [0.1274583786725998, 0.36976999044418335, -0.03472846373915672, -0.3435385525226593], [0.1348537802696228, 0.5653683543205261, -0.04159923642873764, -0.6469672918319702], [0.14616113901138306, 0.3708499073982239, -0.054538581520318985, -0.3676687479019165], [0.15357813239097595, 0.17654363811016083, -0.06189195439219475, -0.09266908466815948], [0.15710900723934174, -0.01763911545276642, -0.06374533474445343, 0.17986272275447845], [0.1567562371492386, -0.2117937058210373, -0.06014808267354965, 0.4517746567726135], [0.15252035856246948, -0.4060157239437103, -0.0511125884950161, 0.7249079942703247], [0.14440004527568817, -0.21022562682628632, -0.03661442920565605, 0.4165858030319214], [0.14019553363323212, -0.4048100411891937, -0.028282713145017624, 0.6975045800209045], [0.13209933042526245, -0.20930756628513336, -0.014332621358335018, 0.39605414867401123], [0.12791317701339722, -0.4042232632637024, -0.006411538925021887, 0.6841840147972107], [0.1198287084698677, -0.20901288092136383, 0.007272141519933939, 0.38948947191238403], [0.11564845591783524, -0.013994891196489334, 0.015061930753290653, 0.09910821169614792], [0.11536855250597, -0.20932942628860474, 0.01704409532248974, 0.39650481939315796], [0.11118196696043015, -0.014453399926424026, 0.024974191561341286, 0.1092439591884613], [0.1108928993344307, 0.18030193448066711, 0.0271590705960989, -0.17545630037784576], [0.11449893563985825, 0.3750248849391937, 0.023649943992495537, -0.45944923162460327], [0.12199943512678146, 0.5698046684265137, 0.014460960403084755, -0.7445847988128662], [0.13339552283287048, 0.7647241353988647, -0.000430735235568136, -1.032681941986084], [0.14869001507759094, 0.9598518013954163, -0.02108437567949295, -1.3255001306533813], [0.16788704693317413, 0.7650023102760315, -0.047594375908374786, -1.0394892692565918], [0.1831870973110199, 0.5705440044403076, -0.06838416308164597, -0.762119472026825], [0.1945979744195938, 0.7665379047393799, -0.08362655341625214, -1.0755125284194946], [0.2099287360906601, 0.5726144313812256, -0.10513680428266525, -0.8102014064788818], [0.2213810235261917, 0.3790779709815979, -0.12134082615375519, -0.552353024482727], [0.22896258533000946, 0.1858503371477127, -0.1323878914117813, -0.30022940039634705], [0.2326795905828476, -0.007160619366914034, -0.13839247822761536, -0.05205482989549637], [0.23253637552261353, -0.2000548243522644, -0.1394335776567459, 0.1939624696969986], [0.22853527963161469, -0.3929353952407837, -0.13555432856082916, 0.4396146833896637], [0.22067657113075256, -0.1961815357208252, -0.1267620325088501, 0.10746019333600998], [0.21675294637680054, 0.0005075104418210685, -0.12461283057928085, -0.22237446904182434], [0.21676309406757355, 0.19716985523700714, -0.1290603131055832, -0.5516219735145569], [0.2207064926624298, 0.39384564757347107, -0.14009276032447815, -0.8820206522941589], [0.22858339548110962, 0.5905638933181763, -0.15773317217826843, -1.2152597904205322], [0.2403946816921234, 0.39778846502304077, -0.18203836679458618, -0.9758678674697876], [0.24835044145584106, 0.20551282167434692, -0.20155572891235352, -0.745444118976593], [0.2524607181549072, 0.01365789957344532, -0.21646460890769958, -0.5223444700241089]]} +{"type": "SampleBatch", "weights": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "eps_id": [1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020, 1238833020], "dones": [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, true], "infos": [{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}], "prev_rewards": [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "t": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25], "agent_index": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "action_prob": [0.5135254263877869, 0.4770704507827759, 0.5442214012145996, 0.47627949714660645, 0.5454674363136292, 0.5253314971923828, 0.48434364795684814, 0.5828204154968262, 0.48531463742256165, 0.5827109813690186, 0.5136748552322388, 0.4766709804534912, 0.45407694578170776, 0.4279625415802002, 0.5955550074577332, 0.5748928189277649, 0.5481062531471252, 0.4735119938850403, 0.5489782094955444, 0.47440415620803833, 0.5505622625350952, 0.5247683525085449, 0.5148704051971436, 0.4746163487434387, 0.4442490339279175, 0.4205590784549713], "actions": [1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1], "q_values": [[-0.015597449615597725, 0.038517292588949203], [0.04316295310854912, -0.04861947521567345], [0.09876783937215805, -0.0785810723900795], [0.03863132745027542, -0.05632191151380539], [0.09450361132621765, -0.08787006139755249], [0.033118072897195816, -0.06829479336738586], [-0.011613234877586365, 0.0510326623916626], [-0.08389873802661896, 0.25046348571777344], [-0.021378351375460625, 0.0373799204826355], [-0.08555285632610321, 0.24835921823978424], [-0.028901388868689537, 0.025811681523919106], [0.02785981446504593, -0.0655241534113884], [0.0917566642165184, -0.09245472401380539], [0.1692613959312439, -0.12090739607810974], [0.25693047046661377, -0.1300475001335144], [0.1545487344264984, -0.14729353785514832], [0.055337414145469666, -0.13768470287322998], [0.00671960785984993, -0.09933169186115265], [0.05141502618789673, -0.14512820541858673], [-0.008995093405246735, -0.1114681214094162], [0.0450827032327652, -0.15785999596118927], [-0.02486952394247055, -0.12402410060167313], [-0.15750475227832794, -0.09800545871257782], [-0.04371977970004082, -0.14534175395965576], [0.03489668667316437, -0.1890382468700409], [0.1171964704990387, -0.20328232645988464]], "rewards": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "prev_actions": [0, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1], "obs": [[0.0450199730694294, -0.03486160933971405, 0.016064710915088654, 0.011697827838361263], [0.04432274028658867, 0.16002631187438965, 0.01629866659641266, -0.2758735120296478], [0.047523267567157745, 0.3549119830131531, 0.010781196877360344, -0.5633715987205505], [0.05462150648236275, 0.15964041650295258, -0.0004862352798227221, -0.2673116624355316], [0.05781431496143341, 0.3547693192958832, -0.0058324686251580715, -0.5601479411125183], [0.06490969657897949, 0.1597297042608261, -0.017035426571965218, -0.2693082094192505], [0.06810429692268372, -0.035145051777362823, -0.022421590983867645, 0.01795332506299019], [0.06740139424800873, -0.22993838787078857, -0.022062525153160095, 0.30347850918769836], [0.06280262768268585, -0.03450907766819, -0.01599295437335968, 0.00392001261934638], [0.06211244314908981, -0.22939805686473846, -0.0159145537763834, 0.2915143668651581], [0.057524483650922775, -0.03405284881591797, -0.010084266774356365, -0.006145021412521601], [0.05684342607855797, 0.1612122654914856, -0.010207167826592922, -0.3019925057888031], [0.06006767228245735, 0.35647818446159363, -0.016247017309069633, -0.597877025604248], [0.06719723343849182, 0.5518236756324768, -0.028204558417201042, -0.8956329822540283], [0.07823371142148972, 0.7473164796829224, -0.04611721634864807, -1.1970465183258057], [0.09318003803491592, 0.5528207421302795, -0.07005815207958221, -0.9191668629646301], [0.10423645377159119, 0.35871216654777527, -0.08844148367643356, -0.6492984294891357], [0.11141069233417511, 0.16492627561092377, -0.10142745822668076, -0.38572362065315247], [0.11470922082662582, 0.3613308370113373, -0.10914192348718643, -0.7085849642753601], [0.12193583697080612, 0.16787634789943695, -0.12331362813711166, -0.45215386152267456], [0.12529335916042328, 0.36450672149658203, -0.1323567032814026, -0.7810221314430237], [0.13258349895477295, 0.1714283674955368, -0.14797714352607727, -0.5327370762825012], [0.13601206243038177, -0.021336432546377182, -0.15863189101219177, -0.29009655117988586], [0.13558533787727356, 0.17564991116523743, -0.1644338220357895, -0.6283085346221924], [0.13909833133220673, 0.3726385235786438, -0.1769999861717224, -0.9679317474365234], [0.14655110239982605, 0.5696383714675903, -0.19635862112045288, -1.3105814456939697]], "new_obs": [[0.04432274028658867, 0.16002631187438965, 0.01629866659641266, -0.2758735120296478], [0.047523267567157745, 0.3549119830131531, 0.010781196877360344, -0.5633715987205505], [0.05462150648236275, 0.15964041650295258, -0.0004862352798227221, -0.2673116624355316], [0.05781431496143341, 0.3547693192958832, -0.0058324686251580715, -0.5601479411125183], [0.06490969657897949, 0.1597297042608261, -0.017035426571965218, -0.2693082094192505], [0.06810429692268372, -0.035145051777362823, -0.022421590983867645, 0.01795332506299019], [0.06740139424800873, -0.22993838787078857, -0.022062525153160095, 0.30347850918769836], [0.06280262768268585, -0.03450907766819, -0.01599295437335968, 0.00392001261934638], [0.06211244314908981, -0.22939805686473846, -0.0159145537763834, 0.2915143668651581], [0.057524483650922775, -0.03405284881591797, -0.010084266774356365, -0.006145021412521601], [0.05684342607855797, 0.1612122654914856, -0.010207167826592922, -0.3019925057888031], [0.06006767228245735, 0.35647818446159363, -0.016247017309069633, -0.597877025604248], [0.06719723343849182, 0.5518236756324768, -0.028204558417201042, -0.8956329822540283], [0.07823371142148972, 0.7473164796829224, -0.04611721634864807, -1.1970465183258057], [0.09318003803491592, 0.5528207421302795, -0.07005815207958221, -0.9191668629646301], [0.10423645377159119, 0.35871216654777527, -0.08844148367643356, -0.6492984294891357], [0.11141069233417511, 0.16492627561092377, -0.10142745822668076, -0.38572362065315247], [0.11470922082662582, 0.3613308370113373, -0.10914192348718643, -0.7085849642753601], [0.12193583697080612, 0.16787634789943695, -0.12331362813711166, -0.45215386152267456], [0.12529335916042328, 0.36450672149658203, -0.1323567032814026, -0.7810221314430237], [0.13258349895477295, 0.1714283674955368, -0.14797714352607727, -0.5327370762825012], [0.13601206243038177, -0.021336432546377182, -0.15863189101219177, -0.29009655117988586], [0.13558533787727356, 0.17564991116523743, -0.1644338220357895, -0.6283085346221924], [0.13909833133220673, 0.3726385235786438, -0.1769999861717224, -0.9679317474365234], [0.14655110239982605, 0.5696383714675903, -0.19635862112045288, -1.3105814456939697], [0.15794387459754944, 0.7666289806365967, -0.2225702553987503, -1.6577483415603638]]} +{"type": "SampleBatch", "weights": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "eps_id": [464626363, 464626363, 464626363, 464626363, 464626363, 464626363, 464626363, 464626363, 464626363, 464626363, 464626363, 464626363, 464626363, 464626363, 464626363, 464626363, 464626363, 464626363, 464626363, 464626363], "dones": [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, true], "infos": [{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}], "prev_rewards": [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "t": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], "agent_index": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "action_prob": [0.49811699986457825, 0.5603018999099731, 0.4948766827583313, 0.5607614517211914, 0.4922669231891632, 0.43934890627861023, 0.6127749681472778, 0.438413143157959, 0.38857191801071167, 0.6461699604988098, 0.6107516288757324, 0.43830615282058716, 0.608411967754364, 0.5631444454193115, 0.518650472164154, 0.5026047825813293, 0.48087823390960693, 0.5650154948234558, 0.4770132005214691, 0.5669832229614258], "actions": [0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1], "q_values": [[0.034373246133327484, 0.041905246675014496], [-0.040324486792087555, 0.20206278562545776], [0.03108956664800644, 0.0515836626291275], [-0.03812238574028015, 0.20613068342208862], [0.016220448538661003, 0.047155141830444336], [-0.03483893722295761, 0.20896606147289276], [-0.10473792254924774, 0.3542538285255432], [-0.02594645321369171, 0.22165822982788086], [-0.10031923651695251, 0.35299989581108093], [-0.1714298129081726, 0.430816113948822], [-0.09505866467952728, 0.3554142117500305], [0.0006859749555587769, 0.2487252801656723], [-0.08787457644939423, 0.35276734828948975], [0.004122734069824219, 0.25805625319480896], [0.038704317063093185, 0.11334069073200226], [-0.01853189617395401, -0.028951097279787064], [0.025288723409175873, 0.10181311517953873], [-0.020684152841567993, 0.24085858464241028], [0.013561476022005081, 0.10557354986667633], [-0.03565507382154465, 0.23389792442321777]], "rewards": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "prev_actions": [0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0], "obs": [[-0.03543581813573837, 0.03231120854616165, 0.04250812903046608, -0.04545578733086586], [-0.03478959575295448, -0.16339369118213654, 0.04159901291131973, 0.2603300213813782], [-0.038057468831539154, 0.0311104916036129, 0.04680561274290085, -0.018947282806038857], [-0.03743525967001915, -0.16465036571025848, 0.046426668763160706, 0.28812822699546814], [-0.04072826728224754, 0.02977983094751835, 0.052189234644174576, 0.010441737249493599], [-0.04013266786932945, -0.1660502403974533, 0.052398066967725754, 0.3191235661506653], [-0.043453674763441086, -0.36187776923179626, 0.05878053978085518, 0.6278597116470337], [-0.05069122835993767, -0.1676233410835266, 0.0713377296924591, 0.35425281524658203], [-0.05404369533061981, -0.36368328332901, 0.07842279225587845, 0.6685502529144287], [-0.061317361891269684, -0.5598031282424927, 0.09179379791021347, 0.9848584532737732], [-0.07251342386007309, -0.36602261662483215, 0.11149096488952637, 0.7223610281944275], [-0.07983388006687164, -0.17260494828224182, 0.12593817710876465, 0.4667462706565857], [-0.08328597247600555, -0.3692602813243866, 0.13527311384677887, 0.7963211536407471], [-0.09067118167877197, -0.17622822523117065, 0.15119953453540802, 0.5490673184394836], [-0.09419574588537216, 0.01648259162902832, 0.16218088567256927, 0.30758246779441833], [-0.09386609494686127, 0.20896689593791962, 0.1683325320482254, 0.0701172798871994], [-0.08968675881624222, 0.011881737969815731, 0.1697348803281784, 0.4108228385448456], [-0.08944912254810333, -0.18518869578838348, 0.17795133590698242, 0.751843273639679], [-0.09315289556980133, 0.00709147984161973, 0.19298820197582245, 0.5200196504592896], [-0.09301106631755829, -0.1901485174894333, 0.20338858664035797, 0.8667741417884827]], "new_obs": [[-0.03478959575295448, -0.16339369118213654, 0.04159901291131973, 0.2603300213813782], [-0.038057468831539154, 0.0311104916036129, 0.04680561274290085, -0.018947282806038857], [-0.03743525967001915, -0.16465036571025848, 0.046426668763160706, 0.28812822699546814], [-0.04072826728224754, 0.02977983094751835, 0.052189234644174576, 0.010441737249493599], [-0.04013266786932945, -0.1660502403974533, 0.052398066967725754, 0.3191235661506653], [-0.043453674763441086, -0.36187776923179626, 0.05878053978085518, 0.6278597116470337], [-0.05069122835993767, -0.1676233410835266, 0.0713377296924591, 0.35425281524658203], [-0.05404369533061981, -0.36368328332901, 0.07842279225587845, 0.6685502529144287], [-0.061317361891269684, -0.5598031282424927, 0.09179379791021347, 0.9848584532737732], [-0.07251342386007309, -0.36602261662483215, 0.11149096488952637, 0.7223610281944275], [-0.07983388006687164, -0.17260494828224182, 0.12593817710876465, 0.4667462706565857], [-0.08328597247600555, -0.3692602813243866, 0.13527311384677887, 0.7963211536407471], [-0.09067118167877197, -0.17622822523117065, 0.15119953453540802, 0.5490673184394836], [-0.09419574588537216, 0.01648259162902832, 0.16218088567256927, 0.30758246779441833], [-0.09386609494686127, 0.20896689593791962, 0.1683325320482254, 0.0701172798871994], [-0.08968675881624222, 0.011881737969815731, 0.1697348803281784, 0.4108228385448456], [-0.08944912254810333, -0.18518869578838348, 0.17795133590698242, 0.751843273639679], [-0.09315289556980133, 0.00709147984161973, 0.19298820197582245, 0.5200196504592896], [-0.09301106631755829, -0.1901485174894333, 0.20338858664035797, 0.8667741417884827], [-0.09681403636932373, 0.0017116105882450938, 0.22072407603263855, 0.6443008184432983]]} diff --git a/python/ray/rllib/tests/run_silent.sh b/python/ray/rllib/tests/run_silent.sh deleted file mode 100755 index 0ddca2b9c..000000000 --- a/python/ray/rllib/tests/run_silent.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -TMPFILE=`mktemp` -DIRECTORY=`dirname $0` -SCRIPT=$1 -shift - -if [ -x $DIRECTORY/../$SCRIPT ]; then - time $DIRECTORY/../$SCRIPT "$@" >$TMPFILE 2>&1 -else - time python $DIRECTORY/../$SCRIPT "$@" >$TMPFILE 2>&1 -fi - -CODE=$? -if [ $CODE != 0 ]; then - cat $TMPFILE - echo "FAILED $CODE" - exit $CODE -fi - -exit 0