diff --git a/ci/travis/ci.sh b/ci/travis/ci.sh
index 600311255..8a0b5ff0f 100755
--- a/ci/travis/ci.sh
+++ b/ci/travis/ci.sh
@@ -141,6 +141,7 @@ test_python() {
       -python/ray/tests:test_object_manager
       -python/ray/tests:test_projects
       -python/ray/tests:test_queue  # timeout
+      -python/ray/tests:test_ray_init  # flaky
       -python/ray/tests:test_reconstruction  # UnreconstructableError
       -python/ray/tests:test_stress
       -python/ray/tests:test_stress_sharded
diff --git a/python/ray/tests/BUILD b/python/ray/tests/BUILD
index 611920652..7ebb8a935 100644
--- a/python/ray/tests/BUILD
+++ b/python/ray/tests/BUILD
@@ -194,14 +194,14 @@ py_test(
 py_test(
     name = "test_autoscaler_aws",
     size = "small",
-    srcs = ["aws/test_autoscaler_aws.py"],
+    srcs = SRCS + ["aws/test_autoscaler_aws.py"],
     deps = ["//:ray_lib"],
 )
 
 py_test(
     name = "test_autoscaler_yaml",
     size = "small",
-    srcs = SRCS + ["test_autoscaler_yaml.py"],
+    srcs = SRCS + ["test_autoscaler_yaml.py", "additional_property.yaml"],
     deps = ["//:ray_lib"],
 )