diff --git a/java/test/pom.xml b/java/test/pom.xml index 10346caf7..e3c6e8a46 100644 --- a/java/test/pom.xml +++ b/java/test/pom.xml @@ -80,6 +80,9 @@ maven-surefire-plugin 2.21.0 + + ../testng.xml + false ${basedir}/src/main/java/ ${project.build.directory}/classes/ diff --git a/java/test/pom_template.xml b/java/test/pom_template.xml index 64ac02b2c..01db3d5a8 100644 --- a/java/test/pom_template.xml +++ b/java/test/pom_template.xml @@ -36,6 +36,9 @@ maven-surefire-plugin 2.21.0 + + ../testng.xml + false ${basedir}/src/main/java/ ${project.build.directory}/classes/ diff --git a/java/test/src/main/java/io/ray/test/ActorConcurrentCallTest.java b/java/test/src/main/java/io/ray/test/ActorConcurrentCallTest.java index c1f963bf2..3eac9fcdc 100644 --- a/java/test/src/main/java/io/ray/test/ActorConcurrentCallTest.java +++ b/java/test/src/main/java/io/ray/test/ActorConcurrentCallTest.java @@ -9,7 +9,7 @@ import java.util.concurrent.CountDownLatch; import org.testng.Assert; import org.testng.annotations.Test; -@Test +@Test(groups = {"cluster"}) public class ActorConcurrentCallTest extends BaseTest { public static class ConcurrentActor { @@ -27,8 +27,6 @@ public class ActorConcurrentCallTest extends BaseTest { } public void testConcurrentCall() { - TestUtils.skipTestUnderSingleProcess(); - ActorHandle actor = Ray.actor(ConcurrentActor::new) .setMaxConcurrency(3) diff --git a/java/test/src/main/java/io/ray/test/ActorRestartTest.java b/java/test/src/main/java/io/ray/test/ActorRestartTest.java index 1c7fdb946..132c8954c 100644 --- a/java/test/src/main/java/io/ray/test/ActorRestartTest.java +++ b/java/test/src/main/java/io/ray/test/ActorRestartTest.java @@ -13,7 +13,7 @@ import java.util.concurrent.TimeUnit; import org.testng.Assert; import org.testng.annotations.Test; -@Test +@Test(groups = {"cluster"}) public class ActorRestartTest extends BaseTest { public static class Counter { @@ -41,7 +41,6 @@ public class ActorRestartTest extends BaseTest { } public void testActorRestart() throws InterruptedException, IOException { - TestUtils.skipTestUnderSingleProcess(); ActorHandle actor = Ray.actor(Counter::new).setMaxRestarts(1).remote(); // Call increase 3 times. for (int i = 0; i < 3; i++) { @@ -118,7 +117,6 @@ public class ActorRestartTest extends BaseTest { } public void testActorCheckpointing() throws IOException, InterruptedException { - TestUtils.skipTestUnderSingleProcess(); ActorHandle actor = Ray.actor(CheckpointableCounter::new) .setMaxRestarts(1).remote(); // Call increase 3 times. diff --git a/java/test/src/main/java/io/ray/test/ActorTest.java b/java/test/src/main/java/io/ray/test/ActorTest.java index ef1af1ee1..3712da1cf 100644 --- a/java/test/src/main/java/io/ray/test/ActorTest.java +++ b/java/test/src/main/java/io/ray/test/ActorTest.java @@ -118,10 +118,8 @@ public class ActorTest extends BaseTest { } // TODO(qwang): Will re-enable this test case once ref counting is supported in Java. - @Test(enabled = false) + @Test(enabled = false, groups = {"cluster"}) public void testUnreconstructableActorObject() throws InterruptedException { - TestUtils.skipTestUnderSingleProcess(); - // The UnreconstructableException is created by raylet. ActorHandle counter = Ray.actor(Counter::new, 100).remote(); // Call an actor method. diff --git a/java/test/src/main/java/io/ray/test/BaseMultiLanguageTest.java b/java/test/src/main/java/io/ray/test/BaseMultiLanguageTest.java index 3f415653b..a45793ee7 100644 --- a/java/test/src/main/java/io/ray/test/BaseMultiLanguageTest.java +++ b/java/test/src/main/java/io/ray/test/BaseMultiLanguageTest.java @@ -17,10 +17,11 @@ import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; -import org.testng.SkipException; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; +@Test(groups = {"cluster", "multiLanguage"}) public abstract class BaseMultiLanguageTest { private static final Logger LOGGER = LoggerFactory.getLogger(BaseMultiLanguageTest.class); @@ -50,18 +51,8 @@ public abstract class BaseMultiLanguageTest { } } - private void checkMultiLanguageTestFlag() { - if (!"1".equals(System.getenv("ENABLE_MULTI_LANGUAGE_TESTS"))) { - LOGGER.info("Skip Multi-language tests because environment variable " - + "ENABLE_MULTI_LANGUAGE_TESTS isn't set"); - throw new SkipException("Skip test."); - } - } - @BeforeClass(alwaysRun = true) public void setUp() { - checkMultiLanguageTestFlag(); - // Delete existing socket files. for (String socket : ImmutableList.of(RAYLET_SOCKET_NAME, PLASMA_STORE_SOCKET_NAME)) { File file = new File(socket); @@ -117,8 +108,6 @@ public abstract class BaseMultiLanguageTest { @AfterClass(alwaysRun = true) public void tearDown() { - checkMultiLanguageTestFlag(); - // Disconnect to the cluster. Ray.shutdown(); System.clearProperty("ray.redis.address"); diff --git a/java/test/src/main/java/io/ray/test/ClassLoaderTest.java b/java/test/src/main/java/io/ray/test/ClassLoaderTest.java index dd19096e9..e5f43e2d0 100644 --- a/java/test/src/main/java/io/ray/test/ClassLoaderTest.java +++ b/java/test/src/main/java/io/ray/test/ClassLoaderTest.java @@ -38,9 +38,8 @@ public class ClassLoaderTest extends BaseTest { System.clearProperty("ray.job.resource-path"); } - @Test + @Test(groups = {"cluster"}) public void testClassLoaderInMultiThreading() throws Exception { - TestUtils.skipTestUnderSingleProcess(); Assert.assertTrue(TestUtils.getRuntime().getRayConfig().numWorkersPerProcess > 1); final String jobResourcePath = resourcePath + "/" + Ray.getRuntimeContext().getCurrentJobId(); diff --git a/java/test/src/main/java/io/ray/test/DynamicResourceTest.java b/java/test/src/main/java/io/ray/test/DynamicResourceTest.java index aa46f8e62..7b65e941e 100644 --- a/java/test/src/main/java/io/ray/test/DynamicResourceTest.java +++ b/java/test/src/main/java/io/ray/test/DynamicResourceTest.java @@ -15,10 +15,8 @@ public class DynamicResourceTest extends BaseTest { return "hi"; } - @Test + @Test(groups = {"cluster"}) public void testSetResource() { - TestUtils.skipTestUnderSingleProcess(); - // Call a task in advance to warm up the cluster to avoid being too slow to start workers. TestUtils.warmUpCluster(); diff --git a/java/test/src/main/java/io/ray/test/FailureTest.java b/java/test/src/main/java/io/ray/test/FailureTest.java index 7e1dec0ed..803040080 100644 --- a/java/test/src/main/java/io/ray/test/FailureTest.java +++ b/java/test/src/main/java/io/ray/test/FailureTest.java @@ -17,6 +17,7 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +@Test(groups = {"cluster"}) public class FailureTest extends BaseTest { private static final String EXCEPTION_MESSAGE = "Oops"; @@ -84,29 +85,21 @@ public class FailureTest extends BaseTest { } } - @Test public void testNormalTaskFailure() { - TestUtils.skipTestUnderSingleProcess(); assertTaskFailedWithRayTaskException(Ray.task(FailureTest::badFunc).remote()); } - @Test public void testActorCreationFailure() { - TestUtils.skipTestUnderSingleProcess(); ActorHandle actor = Ray.actor(BadActor::new, true).remote(); assertTaskFailedWithRayTaskException(actor.task(BadActor::badMethod).remote()); } - @Test public void testActorTaskFailure() { - TestUtils.skipTestUnderSingleProcess(); ActorHandle actor = Ray.actor(BadActor::new, false).remote(); assertTaskFailedWithRayTaskException(actor.task(BadActor::badMethod).remote()); } - @Test public void testWorkerProcessDying() { - TestUtils.skipTestUnderSingleProcess(); try { Ray.task(FailureTest::badFunc2).remote().get(); Assert.fail("This line shouldn't be reached."); @@ -116,9 +109,7 @@ public class FailureTest extends BaseTest { } } - @Test public void testActorProcessDying() { - TestUtils.skipTestUnderSingleProcess(); ActorHandle actor = Ray.actor(BadActor::new, false).remote(); try { actor.task(BadActor::badMethod2).remote().get(); @@ -136,9 +127,7 @@ public class FailureTest extends BaseTest { } } - @Test public void testGetThrowsQuicklyWhenFoundException() { - TestUtils.skipTestUnderSingleProcess(); List> badFunctions = Arrays.asList(FailureTest::badFunc, FailureTest::badFunc2); TestUtils.warmUpCluster(); diff --git a/java/test/src/main/java/io/ray/test/GcsClientTest.java b/java/test/src/main/java/io/ray/test/GcsClientTest.java index a26f323c3..6a5b2c002 100644 --- a/java/test/src/main/java/io/ray/test/GcsClientTest.java +++ b/java/test/src/main/java/io/ray/test/GcsClientTest.java @@ -11,6 +11,7 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +@Test(groups = {"cluster"}) public class GcsClientTest extends BaseTest { @BeforeClass @@ -23,9 +24,7 @@ public class GcsClientTest extends BaseTest { System.clearProperty("ray.resources"); } - @Test public void testGetAllNodeInfo() { - TestUtils.skipTestUnderSingleProcess(); RayConfig config = TestUtils.getRuntime().getRayConfig(); Preconditions.checkNotNull(config); @@ -39,7 +38,6 @@ public class GcsClientTest extends BaseTest { @Test public void testNextJob() { - TestUtils.skipTestUnderSingleProcess(); RayConfig config = TestUtils.getRuntime().getRayConfig(); // The value of job id of this driver in cluster should be 1. Assert.assertEquals(config.getJobId(), JobId.fromInt(1)); diff --git a/java/test/src/main/java/io/ray/test/KillActorTest.java b/java/test/src/main/java/io/ray/test/KillActorTest.java index 76995322c..c7ac3d673 100644 --- a/java/test/src/main/java/io/ray/test/KillActorTest.java +++ b/java/test/src/main/java/io/ray/test/KillActorTest.java @@ -11,7 +11,7 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -@Test +@Test(groups = {"cluster"}) public class KillActorTest extends BaseTest { @BeforeClass @@ -54,8 +54,6 @@ public class KillActorTest extends BaseTest { } private void testKillActor(BiConsumer, Boolean> kill, boolean noRestart) { - TestUtils.skipTestUnderSingleProcess(); - ActorHandle actor = Ray.actor(HangActor::new) .setMaxRestarts(1) .remote(); diff --git a/java/test/src/main/java/io/ray/test/MetricTest.java b/java/test/src/main/java/io/ray/test/MetricTest.java index a439d2102..6e4810945 100644 --- a/java/test/src/main/java/io/ray/test/MetricTest.java +++ b/java/test/src/main/java/io/ray/test/MetricTest.java @@ -17,6 +17,7 @@ import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.Test; +@Test(groups = {"cluster"}) public class MetricTest extends BaseTest { boolean doubleEqual(double value, double other) { @@ -77,9 +78,7 @@ public class MetricTest extends BaseTest { Metrics.shutdown(); } - @Test public void testAddGauge() { - TestUtils.skipTestUnderSingleProcess(); Map tags = new HashMap<>(); tags.put(new TagKey("tag1"), "value1"); @@ -90,9 +89,7 @@ public class MetricTest extends BaseTest { gauge.unregister(); } - @Test public void testAddCount() { - TestUtils.skipTestUnderSingleProcess(); Map tags = new HashMap<>(); tags.put(new TagKey("tag1"), "value1"); tags.put(new TagKey("count_tag"), "default"); @@ -104,9 +101,7 @@ public class MetricTest extends BaseTest { Assert.assertTrue(doubleEqual(count.getCount(), 30.0)); } - @Test public void testAddSum() { - TestUtils.skipTestUnderSingleProcess(); Map tags = new HashMap<>(); tags.put(new TagKey("tag1"), "value1"); tags.put(new TagKey("sum_tag"), "default"); @@ -118,9 +113,7 @@ public class MetricTest extends BaseTest { Assert.assertTrue(doubleEqual(sum.getSum(), 30.0)); } - @Test public void testAddHistogram() { - TestUtils.skipTestUnderSingleProcess(); Map tags = new HashMap<>(); tags.put(new TagKey("tag1"), "value1"); tags.put(new TagKey("histogram_tag"), "default"); diff --git a/java/test/src/main/java/io/ray/test/MultiThreadingTest.java b/java/test/src/main/java/io/ray/test/MultiThreadingTest.java index c03d9f941..7a63c6435 100644 --- a/java/test/src/main/java/io/ray/test/MultiThreadingTest.java +++ b/java/test/src/main/java/io/ray/test/MultiThreadingTest.java @@ -121,15 +121,15 @@ public class MultiThreadingTest extends BaseTest { testMultiThreading(); } + // Single-process mode doesn't have real workers. + @Test(groups = {"cluster"}) public void testInWorker() { - // Single-process mode doesn't have real workers. - TestUtils.skipTestUnderSingleProcess(); ObjectRef obj = Ray.task(MultiThreadingTest::testMultiThreading).remote(); Assert.assertEquals("ok", obj.get()); } + @Test(groups = {"cluster"}) public void testGetCurrentActorId() { - TestUtils.skipTestUnderSingleProcess(); ActorHandle actorIdTester = Ray.actor(ActorIdTester::new).remote(); ActorId actorId = actorIdTester.task(ActorIdTester::getCurrentActorId).remote().get(); Assert.assertEquals(actorId, actorIdTester.getId()); diff --git a/java/test/src/main/java/io/ray/test/PlasmaFreeTest.java b/java/test/src/main/java/io/ray/test/PlasmaFreeTest.java index b3441505f..99adafc7a 100644 --- a/java/test/src/main/java/io/ray/test/PlasmaFreeTest.java +++ b/java/test/src/main/java/io/ray/test/PlasmaFreeTest.java @@ -32,9 +32,8 @@ public class PlasmaFreeTest extends BaseTest { } } - @Test + @Test(groups = {"cluster"}) public void testDeleteCreatingTasks() { - TestUtils.skipTestUnderSingleProcess(); ObjectRef helloId = Ray.task(PlasmaFreeTest::hello).remote(); Assert.assertEquals("hello", helloId.get()); Ray.internal().free(ImmutableList.of(helloId.getId()), true, true); diff --git a/java/test/src/main/java/io/ray/test/PlasmaStoreTest.java b/java/test/src/main/java/io/ray/test/PlasmaStoreTest.java index d92a98edd..dcbabf655 100644 --- a/java/test/src/main/java/io/ray/test/PlasmaStoreTest.java +++ b/java/test/src/main/java/io/ray/test/PlasmaStoreTest.java @@ -8,9 +8,8 @@ import org.testng.annotations.Test; public class PlasmaStoreTest extends BaseTest { - @Test + @Test(groups = {"cluster"}) public void testPutWithDuplicateId() { - TestUtils.skipTestUnderSingleProcess(); ObjectId objectId = ObjectId.fromRandom(); ObjectStore objectStore = TestUtils.getRuntime().getObjectStore(); objectStore.put("1", objectId); diff --git a/java/test/src/main/java/io/ray/test/RayAlterSuiteListener.java b/java/test/src/main/java/io/ray/test/RayAlterSuiteListener.java new file mode 100644 index 000000000..3544b2dc4 --- /dev/null +++ b/java/test/src/main/java/io/ray/test/RayAlterSuiteListener.java @@ -0,0 +1,27 @@ +package io.ray.test; + +import io.ray.runtime.config.RayConfig; +import io.ray.runtime.config.RunMode; +import java.util.List; +import org.testng.IAlterSuiteListener; +import org.testng.xml.XmlGroups; +import org.testng.xml.XmlRun; +import org.testng.xml.XmlSuite; + +public class RayAlterSuiteListener implements IAlterSuiteListener { + + @Override + public void alter(List suites) { + XmlSuite suite = suites.get(0); + String excludedGroup = + RayConfig.create().runMode == RunMode.SINGLE_PROCESS ? "cluster" : "singleProcess"; + XmlGroups groups = new XmlGroups(); + XmlRun run = new XmlRun(); + run.onExclude(excludedGroup); + if (!"1".equals(System.getenv("ENABLE_MULTI_LANGUAGE_TESTS"))) { + run.onExclude("multiLanguage"); + } + groups.setRun(run); + suite.setGroups(groups); + } +} \ No newline at end of file diff --git a/java/test/src/main/java/io/ray/test/ResourcesManagementTest.java b/java/test/src/main/java/io/ray/test/ResourcesManagementTest.java index 9066da018..68ac20ed6 100644 --- a/java/test/src/main/java/io/ray/test/ResourcesManagementTest.java +++ b/java/test/src/main/java/io/ray/test/ResourcesManagementTest.java @@ -15,6 +15,7 @@ import org.testng.annotations.Test; /** * Resources Management Test. */ +@Test(groups = {"cluster"}) public class ResourcesManagementTest extends BaseTest { @BeforeClass @@ -38,10 +39,7 @@ public class ResourcesManagementTest extends BaseTest { } } - @Test public void testMethods() { - TestUtils.skipTestUnderSingleProcess(); - // This is a case that can satisfy required resources. // The static resources for test are "CPU:4,RES-A:4". ObjectRef result1 = Ray.task(ResourcesManagementTest::echo, 100) @@ -69,10 +67,7 @@ public class ResourcesManagementTest extends BaseTest { } } - @Test public void testActors() { - TestUtils.skipTestUnderSingleProcess(); - // This is a case that can satisfy required resources. // The static resources for test are "CPU:4,RES-A:4". ActorHandle echo1 = Ray.actor(Echo::new) diff --git a/java/test/src/main/java/io/ray/test/SingleProcessModeTest.java b/java/test/src/main/java/io/ray/test/SingleProcessModeTest.java index 2c90e3fa1..af9a4f19a 100644 --- a/java/test/src/main/java/io/ray/test/SingleProcessModeTest.java +++ b/java/test/src/main/java/io/ray/test/SingleProcessModeTest.java @@ -26,10 +26,8 @@ public class SingleProcessModeTest extends BaseTest { } } - @Test + @Test(groups = {"singleProcess"}) public void testActorTasksInOneThread() { - TestUtils.skipTestUnderClusterMode(); - List> actors = new ArrayList<>(); Map actorThreadIds = new HashMap<>(); for (int i = 0; i < NUM_ACTOR_INSTANCE; ++i) { diff --git a/java/test/src/main/java/io/ray/test/StressTest.java b/java/test/src/main/java/io/ray/test/StressTest.java index 280c7d08f..1a5c5e4d9 100644 --- a/java/test/src/main/java/io/ray/test/StressTest.java +++ b/java/test/src/main/java/io/ray/test/StressTest.java @@ -10,15 +10,14 @@ import java.util.List; import org.testng.Assert; import org.testng.annotations.Test; +@Test(groups = {"cluster"}) public class StressTest extends BaseTest { public static int echo(int x) { return x; } - @Test public void testSubmittingTasks() { - TestUtils.skipTestUnderSingleProcess(); for (int numIterations : ImmutableList.of(1, 10, 100, 1000)) { int numTasks = 1000 / numIterations; for (int i = 0; i < numIterations; i++) { @@ -34,9 +33,7 @@ public class StressTest extends BaseTest { } } - @Test public void testDependency() { - TestUtils.skipTestUnderSingleProcess(); ObjectRef x = Ray.task(StressTest::echo, 1).remote(); for (int i = 0; i < 1000; i++) { x = Ray.task(StressTest::echo, x).remote(); @@ -73,9 +70,7 @@ public class StressTest extends BaseTest { } } - @Test public void testSubmittingManyTasksToOneActor() throws Exception { - TestUtils.skipTestUnderSingleProcess(); ActorHandle actor = Ray.actor(Actor::new).remote(); List objectIds = new ArrayList<>(); for (int i = 0; i < 10; i++) { @@ -88,9 +83,7 @@ public class StressTest extends BaseTest { } } - @Test public void testPuttingAndGettingManyObjects() { - TestUtils.skipTestUnderSingleProcess(); Integer objectToPut = 1; List> objects = new ArrayList<>(); for (int i = 0; i < 100_000; i++) { diff --git a/java/test/src/main/java/io/ray/test/TestUtils.java b/java/test/src/main/java/io/ray/test/TestUtils.java index 028f3e409..4dac2ccbb 100644 --- a/java/test/src/main/java/io/ray/test/TestUtils.java +++ b/java/test/src/main/java/io/ray/test/TestUtils.java @@ -8,7 +8,6 @@ import io.ray.runtime.config.RunMode; import java.io.Serializable; import java.util.function.Supplier; import org.testng.Assert; -import org.testng.SkipException; public class TestUtils { @@ -23,18 +22,6 @@ public class TestUtils { return getRuntime().getRayConfig().runMode == RunMode.SINGLE_PROCESS; } - public static void skipTestUnderSingleProcess() { - if (isSingleProcessMode()) { - throw new SkipException("This test doesn't work under single-process mode."); - } - } - - public static void skipTestUnderClusterMode() { - if (getRuntime().getRayConfig().runMode == RunMode.CLUSTER) { - throw new SkipException("This test doesn't work under cluster mode."); - } - } - /** * Wait until the given condition is met. * diff --git a/java/test/src/main/java/io/ray/test/WorkerJvmOptionsTest.java b/java/test/src/main/java/io/ray/test/WorkerJvmOptionsTest.java index d9ec74b9e..e3134627b 100644 --- a/java/test/src/main/java/io/ray/test/WorkerJvmOptionsTest.java +++ b/java/test/src/main/java/io/ray/test/WorkerJvmOptionsTest.java @@ -14,9 +14,8 @@ public class WorkerJvmOptionsTest extends BaseTest { } } - @Test + @Test(groups = {"cluster"}) public void testJvmOptions() { - TestUtils.skipTestUnderSingleProcess(); // The whitespaces in following argument are intentionally added to test // that raylet can correctly handle dynamic options with whitespaces. ActorHandle actor = Ray.actor(Echo::new) diff --git a/java/testng.xml b/java/testng.xml index 20ed7c3ee..d81b75252 100644 --- a/java/testng.xml +++ b/java/testng.xml @@ -8,6 +8,7 @@ +