wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-01 11:27:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Java] Refactor java api (#8858)

Browse Source
This commit is contained in:
chaokunyang
2020-06-12 10:49:01 +08:00
committed by GitHub
parent cae475c46a
commit dfa4768fc6
64 changed files with 1854 additions and 3394 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/client/JobClientImpl.java
+6 -8
View File
@@ -3,7 +3,6 @@ package io.ray.streaming.runtime.client;
import io.ray.api.ActorHandle;
import io.ray.api.ObjectRef;
import io.ray.api.Ray;
import io.ray.api.options.ActorCreationOptions;
import io.ray.streaming.client.JobClient;
import io.ray.streaming.jobgraph.JobGraph;
import io.ray.streaming.runtime.config.global.CommonConfig;
@@ -27,10 +26,6 @@ public class JobClientImpl implements JobClient {
LOG.info("Submitting job [{}] with job graph [{}] and job config [{}].",
jobGraph.getJobName(), jobGraph, jobConfig);
Map<String, Double> resources = new HashMap<>();
ActorCreationOptions options = new ActorCreationOptions.Builder()
.setResources(resources)
.setMaxRestarts(-1)
.createActorCreationOptions();
// set job name and id at start
jobConfig.put(CommonConfig.JOB_ID, Ray.getRuntimeContext().getCurrentJobId().toString());
@@ -39,11 +34,14 @@ public class JobClientImpl implements JobClient {
jobGraph.getJobConfig().putAll(jobConfig);
// create job master actor
this.jobMasterActor = Ray.createActor(JobMaster::new, jobConfig, options);
this.jobMasterActor = Ray.actor(JobMaster::new, jobConfig)
.setResources(resources)
.setMaxRestarts(-1)
.remote();
try {
ObjectRef<Boolean> submitResult = jobMasterActor.call(JobMaster::submitJob,
jobMasterActor, jobGraph);
ObjectRef<Boolean> submitResult = jobMasterActor.task(JobMaster::submitJob,
jobMasterActor, jobGraph).remote();
if (submitResult.get()) {
LOG.info("Finish submitting job: {}.", jobGraph.getJobName());
streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/scheduler/controller/WorkerLifecycleController.java
+9 -9
View File
@@ -6,7 +6,6 @@ import io.ray.api.Ray;
import io.ray.api.WaitResult;
import io.ray.api.function.PyActorClass;
import io.ray.api.id.ActorId;
import io.ray.api.options.ActorCreationOptions;
import io.ray.streaming.api.Language;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionVertex;
@@ -46,17 +45,18 @@ public class WorkerLifecycleController {
Language language = executionVertex.getLanguage();
ActorCreationOptions options = new ActorCreationOptions.Builder()
.setResources(executionVertex.getResource())
.setMaxRestarts(-1)
.createActorCreationOptions();
BaseActorHandle actor;
if (Language.JAVA == language) {
actor = Ray.createActor(JobWorker::new, options);
actor = Ray.actor(JobWorker::new)
.setResources(executionVertex.getResource())
.setMaxRestarts(-1)
.remote();
} else {
actor = Ray.createActor(
new PyActorClass("ray.streaming.runtime.worker", "JobWorker"));
actor = Ray.actor(new PyActorClass(
"ray.streaming.runtime.worker", "JobWorker"))
.setResources(executionVertex.getResource())
.setMaxRestarts(-1)
.remote();
}
if (null == actor) {
streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/RemoteCallWorker.java
+6 -5
View File
@@ -32,11 +32,11 @@ public class RemoteCallWorker {
// python
if (actor instanceof PyActorHandle) {
result = ((PyActorHandle) actor).call(
new PyActorMethod("init", Object.class), context.getPythonWorkerContextBytes());
result = ((PyActorHandle) actor).task(new PyActorMethod("init", Object.class),
context.getPythonWorkerContextBytes()).remote();
} else {
// java
result = ((ActorHandle<JobWorker>) actor).call(JobWorker::init, context);
result = ((ActorHandle<JobWorker>) actor).task(JobWorker::init, context).remote();
}
LOG.info("Finished calling worker to initiate.");
@@ -55,10 +55,11 @@ public class RemoteCallWorker {
// python
if (actor instanceof PyActorHandle) {
result = ((PyActorHandle) actor).call(new PyActorMethod("start", Object.class));
result = ((PyActorHandle) actor)
.task(new PyActorMethod("start", Object.class)).remote();
} else {
// java
result = ((ActorHandle<JobWorker>) actor).call(JobWorker::start);
result = ((ActorHandle<JobWorker>) actor).task(JobWorker::start).remote();
}
LOG.info("Finished calling worker to start.");
streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/streamingqueue/StreamingQueueTest.java
+9 -16
View File
@@ -3,8 +3,6 @@ package io.ray.streaming.runtime.streamingqueue;
import com.google.common.collect.ImmutableMap;
import io.ray.api.ActorHandle;
import io.ray.api.Ray;
import io.ray.api.options.ActorCreationOptions;
import io.ray.api.options.ActorCreationOptions.Builder;
import io.ray.runtime.config.RayConfig;
import io.ray.streaming.api.context.StreamingContext;
import io.ray.streaming.api.function.impl.FlatMapFunction;
@@ -30,7 +28,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@@ -89,19 +86,15 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
// ray init
Ray.init();
ActorCreationOptions.Builder builder = new Builder();
ActorHandle<WriterWorker> writerActor = Ray.createActor(WriterWorker::new, "writer",
builder.createActorCreationOptions());
ActorHandle<ReaderWorker> readerActor = Ray.createActor(ReaderWorker::new, "reader",
builder.createActorCreationOptions());
ActorHandle<WriterWorker> writerActor = Ray.actor(WriterWorker::new, "writer").remote();
ActorHandle<ReaderWorker> readerActor = Ray.actor(ReaderWorker::new, "reader").remote();
LOGGER.info("call getName on writerActor: {}",
writerActor.call(WriterWorker::getName).get());
writerActor.task(WriterWorker::getName).remote().get());
LOGGER.info("call getName on readerActor: {}",
readerActor.call(ReaderWorker::getName).get());
readerActor.task(ReaderWorker::getName).remote().get());
// LOGGER.info(writerActor.call(WriterWorker::testCallReader, readerActor).get());
// LOGGER.info(writerActor.task(WriterWorker::testCallReader, readerActor).remote().get());
List<String> outputQueueList = new ArrayList<>();
List<String> inputQueueList = new ArrayList<>();
int queueNum = 2;
@@ -114,17 +107,17 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
}
final int msgCount = 100;
readerActor.call(ReaderWorker::init, inputQueueList, writerActor, msgCount);
readerActor.task(ReaderWorker::init, inputQueueList, writerActor, msgCount).remote();
try {
Thread.sleep(1000);
} catch (Exception e) {
e.printStackTrace();
}
writerActor.call(WriterWorker::init, outputQueueList, readerActor, msgCount);
writerActor.task(WriterWorker::init, outputQueueList, readerActor, msgCount).remote();
long time = 0;
while (time < 20000 &&
readerActor.call(ReaderWorker::getTotalMsg).get() < msgCount * queueNum) {
readerActor.task(ReaderWorker::getTotalMsg).remote().get() < msgCount * queueNum) {
try {
Thread.sleep(1000);
time += 1000;
@@ -134,7 +127,7 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
}
Assert.assertEquals(
readerActor.call(ReaderWorker::getTotalMsg).get().intValue(),
readerActor.task(ReaderWorker::getTotalMsg).remote().get().intValue(),
msgCount * queueNum);
}
streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/streamingqueue/Worker.java
+2 -2
View File
@@ -191,7 +191,7 @@ class WriterWorker extends Worker {
}
public String testCallReader(ActorHandle<ReaderWorker> readerActor) {
String name = readerActor.call(ReaderWorker::getName).get();
String name = readerActor.task(ReaderWorker::getName).remote().get();
LOGGER.info("testCallReader: {}", name);
return name;
}
@@ -211,7 +211,7 @@ class WriterWorker extends Worker {
int count = 3;
while (count-- != 0) {
peer.call(ReaderWorker::testRayCall).get();
peer.task(ReaderWorker::testRayCall).remote().get();
}
try {