diff --git a/.travis.yml b/.travis.yml
index c5948741c..6e48d406c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -157,15 +157,15 @@ script:
   # - export PYTHONPATH="$PYTHONPATH:./ci/"
 
   # ray tune tests
-  - python python/ray/tune/test/dependency_test.py
+  - python python/ray/tune/tests/test_dependency.py
   # `cluster_tests.py` runs on Jenkins, not Travis.
-  - python -m pytest -v --durations=30 --ignore=python/ray/tune/cluster_tests.py python/ray/tune/test
+  - python -m pytest -v --durations=30 --ignore=python/ray/tune/tests/test_cluster.py python/ray/tune/tests
 
   # ray rllib tests
-  - python python/ray/rllib/test/test_catalog.py
-  - python python/ray/rllib/test/test_filters.py
-  - python python/ray/rllib/test/test_optimizers.py
-  - python python/ray/rllib/test/test_evaluators.py
+  - python python/ray/rllib/tests/test_catalog.py
+  - python python/ray/rllib/tests/test_filters.py
+  - python python/ray/rllib/tests/test_optimizers.py
+  - python python/ray/rllib/tests/test_evaluators.py
 
   # ray tests
   # Python3.5+ only. Otherwise we will get `SyntaxError` regardless of how we set the tester.
diff --git a/ci/jenkins_tests/run_multi_node_tests.sh b/ci/jenkins_tests/run_multi_node_tests.sh
index 0eb284abd..764081655 100755
--- a/ci/jenkins_tests/run_multi_node_tests.sh
+++ b/ci/jenkins_tests/run_multi_node_tests.sh
@@ -80,7 +80,7 @@ python3 $ROOT_DIR/multi_node_docker_test.py \
 ######################## TUNE TESTS #################################
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    pytest /ray/python/ray/tune/test/cluster_tests.py
+    pytest /ray/python/ray/tune/tests/test_cluster.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
     python /ray/python/ray/tune/examples/tune_mnist_ray.py \
diff --git a/ci/jenkins_tests/run_rllib_tests.sh b/ci/jenkins_tests/run_rllib_tests.sh
index dc03cf643..597d82290 100644
--- a/ci/jenkins_tests/run_rllib_tests.sh
+++ b/ci/jenkins_tests/run_rllib_tests.sh
@@ -1,47 +1,47 @@
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env PongDeterministic-v0 \
     --run A3C \
     --stop '{"training_iteration": 2}' \
     --config '{"num_workers": 2}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env Pong-ram-v4 \
     --run A3C \
     --stop '{"training_iteration": 2}' \
     --config '{"num_workers": 2}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env PongDeterministic-v0 \
     --run A2C \
     --stop '{"training_iteration": 2}' \
     --config '{"num_workers": 2}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v1 \
     --run PPO \
     --stop '{"training_iteration": 2}' \
     --config '{"kl_coeff": 1.0, "num_sgd_iter": 10, "lr": 1e-4, "sgd_minibatch_size": 64, "train_batch_size": 2000, "num_workers": 1, "model": {"free_log_std": true}}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v1 \
     --run PPO \
     --stop '{"training_iteration": 2}' \
     --config '{"simple_optimizer": false, "num_sgd_iter": 2, "model": {"use_lstm": true}}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v1 \
     --run PPO \
     --stop '{"training_iteration": 2}' \
     --config '{"simple_optimizer": true, "num_sgd_iter": 2, "model": {"use_lstm": true}}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v1 \
     --run PPO \
     --stop '{"training_iteration": 2}' \
@@ -49,194 +49,194 @@ docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
     --ray-num-gpus 1
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v1 \
     --run PPO \
     --stop '{"training_iteration": 2}' \
     --config '{"kl_coeff": 1.0, "num_sgd_iter": 10, "lr": 1e-4, "sgd_minibatch_size": 64, "train_batch_size": 2000, "num_workers": 1, "use_gae": false, "batch_mode": "complete_episodes"}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v1 \
     --run PPO \
     --stop '{"training_iteration": 2}' \
     --config '{"remote_worker_envs": true, "num_envs_per_worker": 2, "num_workers": 1, "train_batch_size": 100, "sgd_minibatch_size": 50}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env Pendulum-v0 \
     --run ES \
     --stop '{"training_iteration": 2}' \
     --config '{"stepsize": 0.01, "episodes_per_batch": 20, "train_batch_size": 100, "num_workers": 2}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env Pong-v0 \
     --run ES \
     --stop '{"training_iteration": 2}' \
     --config '{"stepsize": 0.01, "episodes_per_batch": 20, "train_batch_size": 100, "num_workers": 2}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v0 \
     --run A3C \
     --stop '{"training_iteration": 2}' \
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v0 \
     --run DQN \
     --stop '{"training_iteration": 2}' \
     --config '{"lr": 1e-3, "schedule_max_timesteps": 100000, "exploration_fraction": 0.1, "exploration_final_eps": 0.02, "dueling": false, "hiddens": [], "model": {"fcnet_hiddens": [64], "fcnet_activation": "relu"}}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v0 \
     --run DQN \
     --stop '{"training_iteration": 2}' \
     --config '{"num_workers": 2}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v0 \
     --run APEX \
     --stop '{"training_iteration": 2}' \
     --config '{"num_workers": 2, "timesteps_per_iteration": 1000, "num_gpus": 0, "min_iter_time_s": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env FrozenLake-v0 \
     --run DQN \
     --stop '{"training_iteration": 2}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env FrozenLake-v0 \
     --run PPO \
     --stop '{"training_iteration": 2}' \
     --config '{"num_sgd_iter": 10, "sgd_minibatch_size": 64, "train_batch_size": 1000, "num_workers": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env PongDeterministic-v4 \
     --run DQN \
     --stop '{"training_iteration": 2}' \
     --config '{"lr": 1e-4, "schedule_max_timesteps": 2000000, "buffer_size": 10000, "exploration_fraction": 0.1, "exploration_final_eps": 0.01, "sample_batch_size": 4, "learning_starts": 10000, "target_network_update_freq": 1000, "gamma": 0.99, "prioritized_replay": true}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env MontezumaRevenge-v0 \
     --run PPO \
     --stop '{"training_iteration": 2}' \
     --config '{"kl_coeff": 1.0, "num_sgd_iter": 10, "lr": 1e-4, "sgd_minibatch_size": 64, "train_batch_size": 2000, "num_workers": 1, "model": {"dim": 40, "conv_filters": [[16, [8, 8], 4], [32, [4, 4], 2], [512, [5, 5], 1]]}}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v1 \
     --run A3C \
     --stop '{"training_iteration": 2}' \
     --config '{"num_workers": 2, "model": {"use_lstm": true}}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v0 \
     --run DQN \
     --stop '{"training_iteration": 2}' \
     --config '{"num_workers": 2}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v0 \
     --run PG \
     --stop '{"training_iteration": 2}' \
     --config '{"sample_batch_size": 500, "num_workers": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v0 \
     --run PG \
     --stop '{"training_iteration": 2}' \
     --config '{"sample_batch_size": 500, "use_pytorch": true}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v0 \
     --run PG \
     --stop '{"training_iteration": 2}' \
     --config '{"sample_batch_size": 500, "num_workers": 1, "model": {"use_lstm": true, "max_seq_len": 100}}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v0 \
     --run PG \
     --stop '{"training_iteration": 2}' \
     --config '{"sample_batch_size": 500, "num_workers": 1, "num_envs_per_worker": 10}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env Pong-v0 \
     --run PG \
     --stop '{"training_iteration": 2}' \
     --config '{"sample_batch_size": 500, "num_workers": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env FrozenLake-v0 \
     --run PG \
     --stop '{"training_iteration": 2}' \
     --config '{"sample_batch_size": 500, "num_workers": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env Pendulum-v0 \
     --run DDPG \
     --stop '{"training_iteration": 2}' \
     --config '{"num_workers": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v0 \
     --run IMPALA \
     --stop '{"training_iteration": 2}' \
     --config '{"num_gpus": 0, "num_workers": 2, "min_iter_time_s": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v0 \
     --run IMPALA \
     --stop '{"training_iteration": 2}' \
     --config '{"num_gpus": 0, "num_workers": 2, "min_iter_time_s": 1, "model": {"use_lstm": true}}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v0 \
     --run IMPALA \
     --stop '{"training_iteration": 2}' \
     --config '{"num_gpus": 0, "num_workers": 2, "min_iter_time_s": 1, "num_data_loader_buffers": 2, "replay_buffer_num_slots": 100, "replay_proportion": 1.0}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v0 \
     --run IMPALA \
     --stop '{"training_iteration": 2}' \
     --config '{"num_gpus": 0, "num_workers": 2, "min_iter_time_s": 1, "num_data_loader_buffers": 2, "replay_buffer_num_slots": 100, "replay_proportion": 1.0, "model": {"use_lstm": true}}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env MountainCarContinuous-v0 \
     --run DDPG \
     --stop '{"training_iteration": 2}' \
     --config '{"num_workers": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env MountainCarContinuous-v0 \
     --run DDPG \
     --stop '{"training_iteration": 2}' \
     --config '{"num_workers": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env Pendulum-v0 \
     --run APEX_DDPG \
     --ray-num-cpus 8 \
@@ -244,7 +244,7 @@ docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
     --config '{"num_workers": 2, "optimizer": {"num_replay_buffer_shards": 1}, "learning_starts": 100, "min_iter_time_s": 1}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env Pendulum-v0 \
     --run APEX_DDPG \
     --ray-num-cpus 8 \
@@ -252,141 +252,141 @@ docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
     --config '{"num_workers": 2, "optimizer": {"num_replay_buffer_shards": 1}, "learning_starts": 100, "min_iter_time_s": 1, "batch_mode": "complete_episodes", "parameter_noise": true}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v0 \
     --run MARWIL \
     --stop '{"training_iteration": 2}' \
-    --config '{"input": "/ray/python/ray/rllib/test/data/cartpole_small", "learning_starts": 0, "input_evaluation": ["wis", "is"], "shuffle_buffer_size": 10}'
+    --config '{"input": "/ray/python/ray/rllib/tests/data/cartpole_small", "learning_starts": 0, "input_evaluation": ["wis", "is"], "shuffle_buffer_size": 10}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v0 \
     --run DQN \
     --stop '{"training_iteration": 2}' \
-    --config '{"input": "/ray/python/ray/rllib/test/data/cartpole_small", "learning_starts": 0, "input_evaluation": ["wis", "is"], "soft_q": true}'
+    --config '{"input": "/ray/python/ray/rllib/tests/data/cartpole_small", "learning_starts": 0, "input_evaluation": ["wis", "is"], "soft_q": true}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh test/test_local.py
+    /ray/python/ray/rllib/tests/run_silent.sh tests/test_local.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh test/test_io.py
+    /ray/python/ray/rllib/tests/run_silent.sh tests/test_io.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh test/test_checkpoint_restore.py
+    /ray/python/ray/rllib/tests/run_silent.sh tests/test_checkpoint_restore.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh test/test_policy_evaluator.py
+    /ray/python/ray/rllib/tests/run_silent.sh tests/test_policy_evaluator.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh test/test_nested_spaces.py
+    /ray/python/ray/rllib/tests/run_silent.sh tests/test_nested_spaces.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh test/test_external_env.py
+    /ray/python/ray/rllib/tests/run_silent.sh tests/test_external_env.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh examples/parametric_action_cartpole.py --run=PG --stop=50
+    /ray/python/ray/rllib/tests/run_silent.sh examples/parametric_action_cartpole.py --run=PG --stop=50
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh examples/parametric_action_cartpole.py --run=PPO --stop=50
+    /ray/python/ray/rllib/tests/run_silent.sh examples/parametric_action_cartpole.py --run=PPO --stop=50
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh examples/parametric_action_cartpole.py --run=DQN --stop=50
+    /ray/python/ray/rllib/tests/run_silent.sh examples/parametric_action_cartpole.py --run=DQN --stop=50
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh test/test_lstm.py
+    /ray/python/ray/rllib/tests/run_silent.sh tests/test_lstm.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh examples/batch_norm_model.py --num-iters=1 --run=PPO
+    /ray/python/ray/rllib/tests/run_silent.sh examples/batch_norm_model.py --num-iters=1 --run=PPO
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh examples/batch_norm_model.py --num-iters=1 --run=PG
+    /ray/python/ray/rllib/tests/run_silent.sh examples/batch_norm_model.py --num-iters=1 --run=PG
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh examples/batch_norm_model.py --num-iters=1 --run=DQN
+    /ray/python/ray/rllib/tests/run_silent.sh examples/batch_norm_model.py --num-iters=1 --run=DQN
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh examples/batch_norm_model.py --num-iters=1 --run=DDPG
+    /ray/python/ray/rllib/tests/run_silent.sh examples/batch_norm_model.py --num-iters=1 --run=DDPG
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh test/test_multi_agent_env.py
+    /ray/python/ray/rllib/tests/run_silent.sh tests/test_multi_agent_env.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh test/test_supported_spaces.py
+    /ray/python/ray/rllib/tests/run_silent.sh tests/test_supported_spaces.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    pytest /ray/python/ray/tune/test/cluster_tests.py
+    pytest /ray/python/ray/tune/tests/test_cluster.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh test/test_env_with_subprocess.py
+    /ray/python/ray/rllib/tests/run_silent.sh tests/test_env_with_subprocess.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh test/test_rollout.sh
+    /ray/python/ray/rllib/tests/run_silent.sh tests/test_rollout.sh
 
 # Run all single-agent regression tests (3x retry each)
 for yaml in $(ls $ROOT_DIR/../../python/ray/rllib/tuned_examples/regression_tests); do
     docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-        /ray/python/ray/rllib/test/run_silent.sh test/run_regression_tests.py \
+        /ray/python/ray/rllib/tests/run_silent.sh tests/run_regression_tests.py \
             /ray/python/ray/rllib/tuned_examples/regression_tests/$yaml
 done
 
 # Try a couple times since it's stochastic
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-        /ray/python/ray/rllib/test/run_silent.sh test/multiagent_pendulum.py || \
+        /ray/python/ray/rllib/tests/run_silent.sh tests/multiagent_pendulum.py || \
     docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-        /ray/python/ray/rllib/test/run_silent.sh test/multiagent_pendulum.py || \
+        /ray/python/ray/rllib/tests/run_silent.sh tests/multiagent_pendulum.py || \
     docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-        /ray/python/ray/rllib/test/run_silent.sh test/multiagent_pendulum.py
+        /ray/python/ray/rllib/tests/run_silent.sh tests/multiagent_pendulum.py
 
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh examples/multiagent_cartpole.py --num-iters=2
+    /ray/python/ray/rllib/tests/run_silent.sh examples/multiagent_cartpole.py --num-iters=2
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh examples/multiagent_two_trainers.py --num-iters=2
+    /ray/python/ray/rllib/tests/run_silent.sh examples/multiagent_two_trainers.py --num-iters=2
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh test/test_avail_actions_qmix.py
+    /ray/python/ray/rllib/tests/run_silent.sh tests/test_avail_actions_qmix.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh examples/cartpole_lstm.py --run=PPO --stop=200
+    /ray/python/ray/rllib/tests/run_silent.sh examples/cartpole_lstm.py --run=PPO --stop=200
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh examples/cartpole_lstm.py --run=IMPALA --stop=100
+    /ray/python/ray/rllib/tests/run_silent.sh examples/cartpole_lstm.py --run=IMPALA --stop=100
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh examples/cartpole_lstm.py --stop=200 --use-prev-action-reward
+    /ray/python/ray/rllib/tests/run_silent.sh examples/cartpole_lstm.py --stop=200 --use-prev-action-reward
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh examples/custom_metrics_and_callbacks.py --num-iters=2
+    /ray/python/ray/rllib/tests/run_silent.sh examples/custom_metrics_and_callbacks.py --num-iters=2
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh contrib/random_agent/random_agent.py
+    /ray/python/ray/rllib/tests/run_silent.sh contrib/random_agent/random_agent.py
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh examples/twostep_game.py --stop=2000 --run=PG
+    /ray/python/ray/rllib/tests/run_silent.sh examples/twostep_game.py --stop=2000 --run=PG
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh examples/twostep_game.py --stop=2000 --run=QMIX
+    /ray/python/ray/rllib/tests/run_silent.sh examples/twostep_game.py --stop=2000 --run=QMIX
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh examples/twostep_game.py --stop=2000 --run=APEX_QMIX
+    /ray/python/ray/rllib/tests/run_silent.sh examples/twostep_game.py --stop=2000 --run=APEX_QMIX
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env PongDeterministic-v4 \
     --run A3C \
     --stop '{"training_iteration": 2}' \
     --config '{"num_workers": 2, "use_pytorch": true, "sample_async": false, "model": {"use_lstm": false, "grayscale": true, "zero_mean": false, "dim": 84}, "preprocessor_pref": "rllib"}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env CartPole-v1 \
     --run A3C \
     --stop '{"training_iteration": 2}' \
     --config '{"num_workers": 2, "use_pytorch": true, "sample_async": false}'
 
 docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    /ray/python/ray/rllib/test/run_silent.sh train.py \
+    /ray/python/ray/rllib/tests/run_silent.sh train.py \
     --env PongDeterministic-v4 \
     --run IMPALA \
     --stop='{"timesteps_total": 40000}' \
diff --git a/doc/source/rllib-examples.rst b/doc/source/rllib-examples.rst
index 87fd2225c..02092ac6c 100644
--- a/doc/source/rllib-examples.rst
+++ b/doc/source/rllib-examples.rst
@@ -30,7 +30,7 @@ Custom Envs and Models
    Example of defining and registering a gym env for use with RLlib.
 - `Registering a custom model with supervised loss <https://github.com/ray-project/ray/blob/master/python/ray/rllib/examples/custom_loss.py>`__:
    Example of defining and registering a custom model with a supervised loss.
-- `Subprocess environment <https://github.com/ray-project/ray/blob/master/python/ray/rllib/test/test_env_with_subprocess.py>`__:
+- `Subprocess environment <https://github.com/ray-project/ray/blob/master/python/ray/rllib/tests/test_env_with_subprocess.py>`__:
    Example of how to ensure subprocesses spawned by envs are killed when RLlib exits.
 - `Batch normalization <https://github.com/ray-project/ray/blob/master/python/ray/rllib/examples/batch_norm_model.py>`__:
    Example of adding batch norm layers to a custom model.
diff --git a/doc/source/rllib-models.rst b/doc/source/rllib-models.rst
index 2abd2b533..54b4613b1 100644
--- a/doc/source/rllib-models.rst
+++ b/doc/source/rllib-models.rst
@@ -133,7 +133,7 @@ Custom TF models should subclass the common RLlib `model class <https://github.c
         },
     })
 
-For a full example of a custom model in code, see the `Carla RLlib model <https://github.com/ray-project/ray/blob/master/python/ray/rllib/examples/carla/models.py>`__ and associated `training scripts <https://github.com/ray-project/ray/tree/master/python/ray/rllib/examples/carla>`__. You can also reference the `unit tests <https://github.com/ray-project/ray/blob/master/python/ray/rllib/test/test_nested_spaces.py>`__ for Tuple and Dict spaces, which show how to access nested observation fields.
+For a full example of a custom model in code, see the `Carla RLlib model <https://github.com/ray-project/ray/blob/master/python/ray/rllib/examples/carla/models.py>`__ and associated `training scripts <https://github.com/ray-project/ray/tree/master/python/ray/rllib/examples/carla>`__. You can also reference the `unit tests <https://github.com/ray-project/ray/blob/master/python/ray/rllib/tests/test_nested_spaces.py>`__ for Tuple and Dict spaces, which show how to access nested observation fields.
 
 Custom Recurrent Models
 ~~~~~~~~~~~~~~~~~~~~~~~
@@ -380,4 +380,4 @@ With a custom policy graph, you can also perform model-based rollouts and option
                 return action_batch
 
 
-If you want take this rollouts data and append it to the sample batch, use the ``add_extra_batch()`` method of the `episode objects <https://github.com/ray-project/ray/blob/master/python/ray/rllib/evaluation/episode.py>`__ passed in. For an example of this, see the ``testReturningModelBasedRolloutsData`` `unit test <https://github.com/ray-project/ray/blob/master/python/ray/rllib/test/test_multi_agent_env.py>`__.
+If you want take this rollouts data and append it to the sample batch, use the ``add_extra_batch()`` method of the `episode objects <https://github.com/ray-project/ray/blob/master/python/ray/rllib/evaluation/episode.py>`__ passed in. For an example of this, see the ``testReturningModelBasedRolloutsData`` `unit test <https://github.com/ray-project/ray/blob/master/python/ray/rllib/tests/test_multi_agent_env.py>`__.
diff --git a/python/ray/rllib/examples/multiagent_cartpole.py b/python/ray/rllib/examples/multiagent_cartpole.py
index e2ab5270f..a00140532 100644
--- a/python/ray/rllib/examples/multiagent_cartpole.py
+++ b/python/ray/rllib/examples/multiagent_cartpole.py
@@ -23,7 +23,7 @@ import ray
 from ray import tune
 from ray.rllib.agents.ppo.ppo_policy_graph import PPOPolicyGraph
 from ray.rllib.models import Model, ModelCatalog
-from ray.rllib.test.test_multi_agent_env import MultiCartpole
+from ray.rllib.tests.test_multi_agent_env import MultiCartpole
 from ray.tune import run_experiments
 from ray.tune.registry import register_env
 
diff --git a/python/ray/rllib/examples/multiagent_two_trainers.py b/python/ray/rllib/examples/multiagent_two_trainers.py
index 46831db45..7bdfb0435 100644
--- a/python/ray/rllib/examples/multiagent_two_trainers.py
+++ b/python/ray/rllib/examples/multiagent_two_trainers.py
@@ -19,7 +19,7 @@ from ray.rllib.agents.dqn.dqn import DQNAgent
 from ray.rllib.agents.dqn.dqn_policy_graph import DQNPolicyGraph
 from ray.rllib.agents.ppo.ppo import PPOAgent
 from ray.rllib.agents.ppo.ppo_policy_graph import PPOPolicyGraph
-from ray.rllib.test.test_multi_agent_env import MultiCartpole
+from ray.rllib.tests.test_multi_agent_env import MultiCartpole
 from ray.tune.logger import pretty_print
 from ray.tune.registry import register_env
 
diff --git a/python/ray/rllib/test/__init__.py b/python/ray/rllib/tests/__init__.py
similarity index 100%
rename from python/ray/rllib/test/__init__.py
rename to python/ray/rllib/tests/__init__.py
diff --git a/python/ray/rllib/test/data/cartpole_small/output-2019-02-03_20-27-20_worker-0_0.json b/python/ray/rllib/tests/data/cartpole_small/output-2019-02-03_20-27-20_worker-0_0.json
similarity index 100%
rename from python/ray/rllib/test/data/cartpole_small/output-2019-02-03_20-27-20_worker-0_0.json
rename to python/ray/rllib/tests/data/cartpole_small/output-2019-02-03_20-27-20_worker-0_0.json
diff --git a/python/ray/rllib/test/mock_evaluator.py b/python/ray/rllib/tests/mock_evaluator.py
similarity index 100%
rename from python/ray/rllib/test/mock_evaluator.py
rename to python/ray/rllib/tests/mock_evaluator.py
diff --git a/python/ray/rllib/test/multiagent_pendulum.py b/python/ray/rllib/tests/multiagent_pendulum.py
similarity index 95%
rename from python/ray/rllib/test/multiagent_pendulum.py
rename to python/ray/rllib/tests/multiagent_pendulum.py
index c4ee5ce76..6317df1c6 100644
--- a/python/ray/rllib/test/multiagent_pendulum.py
+++ b/python/ray/rllib/tests/multiagent_pendulum.py
@@ -5,7 +5,7 @@ from __future__ import division
 from __future__ import print_function
 
 import ray
-from ray.rllib.test.test_multi_agent_env import make_multiagent
+from ray.rllib.tests.test_multi_agent_env import make_multiagent
 from ray.tune import run_experiments
 from ray.tune.registry import register_env
 
diff --git a/python/ray/rllib/test/run_regression_tests.py b/python/ray/rllib/tests/run_regression_tests.py
similarity index 100%
rename from python/ray/rllib/test/run_regression_tests.py
rename to python/ray/rllib/tests/run_regression_tests.py
diff --git a/python/ray/rllib/test/run_silent.sh b/python/ray/rllib/tests/run_silent.sh
similarity index 100%
rename from python/ray/rllib/test/run_silent.sh
rename to python/ray/rllib/tests/run_silent.sh
diff --git a/python/ray/rllib/test/test_avail_actions_qmix.py b/python/ray/rllib/tests/test_avail_actions_qmix.py
similarity index 100%
rename from python/ray/rllib/test/test_avail_actions_qmix.py
rename to python/ray/rllib/tests/test_avail_actions_qmix.py
diff --git a/python/ray/rllib/test/test_catalog.py b/python/ray/rllib/tests/test_catalog.py
similarity index 100%
rename from python/ray/rllib/test/test_catalog.py
rename to python/ray/rllib/tests/test_catalog.py
diff --git a/python/ray/rllib/test/test_checkpoint_restore.py b/python/ray/rllib/tests/test_checkpoint_restore.py
similarity index 100%
rename from python/ray/rllib/test/test_checkpoint_restore.py
rename to python/ray/rllib/tests/test_checkpoint_restore.py
diff --git a/python/ray/rllib/test/test_env_with_subprocess.py b/python/ray/rllib/tests/test_env_with_subprocess.py
similarity index 100%
rename from python/ray/rllib/test/test_env_with_subprocess.py
rename to python/ray/rllib/tests/test_env_with_subprocess.py
diff --git a/python/ray/rllib/test/test_evaluators.py b/python/ray/rllib/tests/test_evaluators.py
similarity index 100%
rename from python/ray/rllib/test/test_evaluators.py
rename to python/ray/rllib/tests/test_evaluators.py
diff --git a/python/ray/rllib/test/test_external_env.py b/python/ray/rllib/tests/test_external_env.py
similarity index 98%
rename from python/ray/rllib/test/test_external_env.py
rename to python/ray/rllib/tests/test_external_env.py
index 7efdd2744..26fd9898b 100644
--- a/python/ray/rllib/test/test_external_env.py
+++ b/python/ray/rllib/tests/test_external_env.py
@@ -13,8 +13,8 @@ from ray.rllib.agents.dqn import DQNAgent
 from ray.rllib.agents.pg import PGAgent
 from ray.rllib.evaluation.policy_evaluator import PolicyEvaluator
 from ray.rllib.env.external_env import ExternalEnv
-from ray.rllib.test.test_policy_evaluator import BadPolicyGraph, \
-    MockPolicyGraph, MockEnv
+from ray.rllib.tests.test_policy_evaluator import (BadPolicyGraph,
+                                                   MockPolicyGraph, MockEnv)
 from ray.tune.registry import register_env
 
 
diff --git a/python/ray/rllib/test/test_filters.py b/python/ray/rllib/tests/test_filters.py
similarity index 98%
rename from python/ray/rllib/test/test_filters.py
rename to python/ray/rllib/tests/test_filters.py
index 664b1388e..f039c6c09 100644
--- a/python/ray/rllib/test/test_filters.py
+++ b/python/ray/rllib/tests/test_filters.py
@@ -8,7 +8,7 @@ import numpy as np
 import ray
 from ray.rllib.utils.filter import RunningStat, MeanStdFilter
 from ray.rllib.utils import FilterManager
-from ray.rllib.test.mock_evaluator import _MockEvaluator
+from ray.rllib.tests.mock_evaluator import _MockEvaluator
 
 
 class RunningStatTest(unittest.TestCase):
diff --git a/python/ray/rllib/test/test_io.py b/python/ray/rllib/tests/test_io.py
similarity index 99%
rename from python/ray/rllib/test/test_io.py
rename to python/ray/rllib/tests/test_io.py
index 6fbb3cd14..ef3969615 100644
--- a/python/ray/rllib/test/test_io.py
+++ b/python/ray/rllib/tests/test_io.py
@@ -19,7 +19,7 @@ from ray.rllib.agents.pg.pg_policy_graph import PGPolicyGraph
 from ray.rllib.evaluation import SampleBatch
 from ray.rllib.offline import IOContext, JsonWriter, JsonReader
 from ray.rllib.offline.json_writer import _to_json
-from ray.rllib.test.test_multi_agent_env import MultiCartpole
+from ray.rllib.tests.test_multi_agent_env import MultiCartpole
 from ray.tune.registry import register_env
 
 SAMPLES = SampleBatch({
diff --git a/python/ray/rllib/test/test_local.py b/python/ray/rllib/tests/test_local.py
similarity index 100%
rename from python/ray/rllib/test/test_local.py
rename to python/ray/rllib/tests/test_local.py
diff --git a/python/ray/rllib/test/test_lstm.py b/python/ray/rllib/tests/test_lstm.py
similarity index 100%
rename from python/ray/rllib/test/test_lstm.py
rename to python/ray/rllib/tests/test_lstm.py
diff --git a/python/ray/rllib/test/test_multi_agent_env.py b/python/ray/rllib/tests/test_multi_agent_env.py
similarity index 98%
rename from python/ray/rllib/test/test_multi_agent_env.py
rename to python/ray/rllib/tests/test_multi_agent_env.py
index 3a2d31453..99f00ccaf 100644
--- a/python/ray/rllib/test/test_multi_agent_env.py
+++ b/python/ray/rllib/tests/test_multi_agent_env.py
@@ -10,10 +10,10 @@ import ray
 from ray.rllib.agents.pg import PGAgent
 from ray.rllib.agents.pg.pg_policy_graph import PGPolicyGraph
 from ray.rllib.agents.dqn.dqn_policy_graph import DQNPolicyGraph
-from ray.rllib.optimizers import SyncSamplesOptimizer, \
-    SyncReplayOptimizer, AsyncGradientsOptimizer
-from ray.rllib.test.test_policy_evaluator import MockEnv, MockEnv2, \
-    MockPolicyGraph
+from ray.rllib.optimizers import (SyncSamplesOptimizer, SyncReplayOptimizer,
+                                  AsyncGradientsOptimizer)
+from ray.rllib.tests.test_policy_evaluator import (MockEnv, MockEnv2,
+                                                   MockPolicyGraph)
 from ray.rllib.evaluation.policy_evaluator import PolicyEvaluator
 from ray.rllib.evaluation.policy_graph import PolicyGraph
 from ray.rllib.evaluation.metrics import collect_metrics
diff --git a/python/ray/rllib/test/test_nested_spaces.py b/python/ray/rllib/tests/test_nested_spaces.py
similarity index 99%
rename from python/ray/rllib/test/test_nested_spaces.py
rename to python/ray/rllib/tests/test_nested_spaces.py
index 772e74a21..bc506152c 100644
--- a/python/ray/rllib/test/test_nested_spaces.py
+++ b/python/ray/rllib/tests/test_nested_spaces.py
@@ -23,7 +23,7 @@ from ray.rllib.models.model import Model
 from ray.rllib.models.pytorch.fcnet import FullyConnectedNetwork
 from ray.rllib.models.pytorch.model import TorchModel
 from ray.rllib.rollout import rollout
-from ray.rllib.test.test_external_env import SimpleServing
+from ray.rllib.tests.test_external_env import SimpleServing
 from ray.tune.registry import register_env
 
 DICT_SPACE = spaces.Dict({
diff --git a/python/ray/rllib/test/test_optimizers.py b/python/ray/rllib/tests/test_optimizers.py
similarity index 99%
rename from python/ray/rllib/test/test_optimizers.py
rename to python/ray/rllib/tests/test_optimizers.py
index 0bb290018..67aba19cf 100644
--- a/python/ray/rllib/test/test_optimizers.py
+++ b/python/ray/rllib/tests/test_optimizers.py
@@ -14,7 +14,7 @@ from ray.rllib.agents.ppo.ppo_policy_graph import PPOPolicyGraph
 from ray.rllib.evaluation import SampleBatch
 from ray.rllib.evaluation.policy_evaluator import PolicyEvaluator
 from ray.rllib.optimizers import AsyncGradientsOptimizer, AsyncSamplesOptimizer
-from ray.rllib.test.mock_evaluator import _MockEvaluator
+from ray.rllib.tests.mock_evaluator import _MockEvaluator
 
 
 class AsyncOptimizerTest(unittest.TestCase):
diff --git a/python/ray/rllib/test/test_policy_evaluator.py b/python/ray/rllib/tests/test_policy_evaluator.py
similarity index 100%
rename from python/ray/rllib/test/test_policy_evaluator.py
rename to python/ray/rllib/tests/test_policy_evaluator.py
diff --git a/python/ray/rllib/test/test_rollout.sh b/python/ray/rllib/tests/test_rollout.sh
similarity index 100%
rename from python/ray/rllib/test/test_rollout.sh
rename to python/ray/rllib/tests/test_rollout.sh
diff --git a/python/ray/rllib/test/test_supported_spaces.py b/python/ray/rllib/tests/test_supported_spaces.py
similarity index 98%
rename from python/ray/rllib/test/test_supported_spaces.py
rename to python/ray/rllib/tests/test_supported_spaces.py
index fd2c7d1eb..7d59a04fb 100644
--- a/python/ray/rllib/test/test_supported_spaces.py
+++ b/python/ray/rllib/tests/test_supported_spaces.py
@@ -9,7 +9,8 @@ import sys
 
 import ray
 from ray.rllib.agents.registry import get_agent_class
-from ray.rllib.test.test_multi_agent_env import MultiCartpole, MultiMountainCar
+from ray.rllib.tests.test_multi_agent_env import (MultiCartpole,
+                                                  MultiMountainCar)
 from ray.rllib.utils.error import UnsupportedSpaceException
 from ray.tune.registry import register_env
 
diff --git a/python/ray/tune/test/automl_searcher_test.py b/python/ray/tune/tests/test_automl_searcher.py
similarity index 100%
rename from python/ray/tune/test/automl_searcher_test.py
rename to python/ray/tune/tests/test_automl_searcher.py
diff --git a/python/ray/tune/test/cluster_tests.py b/python/ray/tune/tests/test_cluster.py
similarity index 100%
rename from python/ray/tune/test/cluster_tests.py
rename to python/ray/tune/tests/test_cluster.py
diff --git a/python/ray/tune/test/dependency_test.py b/python/ray/tune/tests/test_dependency.py
similarity index 100%
rename from python/ray/tune/test/dependency_test.py
rename to python/ray/tune/tests/test_dependency.py
diff --git a/python/ray/tune/test/experiment_test.py b/python/ray/tune/tests/test_experiment.py
similarity index 100%
rename from python/ray/tune/test/experiment_test.py
rename to python/ray/tune/tests/test_experiment.py
diff --git a/python/ray/tune/test/ray_trial_executor_test.py b/python/ray/tune/tests/test_ray_trial_executor.py
similarity index 100%
rename from python/ray/tune/test/ray_trial_executor_test.py
rename to python/ray/tune/tests/test_ray_trial_executor.py
diff --git a/python/ray/tune/test/trial_runner_test.py b/python/ray/tune/tests/test_trial_runner.py
similarity index 100%
rename from python/ray/tune/test/trial_runner_test.py
rename to python/ray/tune/tests/test_trial_runner.py
diff --git a/python/ray/tune/test/trial_scheduler_test.py b/python/ray/tune/tests/test_trial_scheduler.py
similarity index 100%
rename from python/ray/tune/test/trial_scheduler_test.py
rename to python/ray/tune/tests/test_trial_scheduler.py
diff --git a/python/ray/tune/test/tune_server_test.py b/python/ray/tune/tests/test_tune_server.py
similarity index 100%
rename from python/ray/tune/test/tune_server_test.py
rename to python/ray/tune/tests/test_tune_server.py