wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-28 03:02:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

[ID Refactor] Refactor ActorID, TaskID and ObjectID (#5286)

Browse Source 
* Refactor ActorID, TaskID on the Java side.

Left a TODO comment

WIP for ObjectID

ADD test

Fix

Add java part

Fix Java test

Fix

Refine test.

Enable test in CI

* Extra a helper function.

* Resolve TODOs

* Fix Python CI

* Fix Java lint

* Update .travis.yml

Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu>

* Address some comments.

Address some comments.

Add id_specification.rst

Reanme id_specification.rst to id_specification.md

typo

Address zhijun's comments.

Fix test

Address comments.

Fix lint

Address comments

* Fix test

* Address comments.

* Fix build error

* Update src/ray/design_docs/id_specification.md

Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu>

* Update src/ray/design_docs/id_specification.md

Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu>

* Update src/ray/design_docs/id_specification.md

Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu>

* Update src/ray/design_docs/id_specification.md

Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu>

* Update src/ray/design_docs/id_specification.md

Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu>

* Address comments

* Update src/ray/common/id.h

Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu>

* Update src/ray/common/id.h

Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu>

* Update src/ray/common/id.h

Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu>

* Update src/ray/design_docs/id_specification.md

Co-Authored-By: Hao Chen <chenh1024@gmail.com>

* Update src/ray/design_docs/id_specification.md

Co-Authored-By: Hao Chen <chenh1024@gmail.com>

* Address comments.

* Address comments.

* Address comments.

* Update C++ part to make sure task id is generated determantic

* WIP

* Fix core worker

* Fix Java part

* Fix comments.

* Add Python side

* Fix python

* Address comments

* Fix linting

* Fix

* Fix C++ linting

* Add JobId() method to TaskID

* Fix linting

* Update src/ray/common/id.h

Co-Authored-By: Hao Chen <chenh1024@gmail.com>

* Update java/api/src/main/java/org/ray/api/id/TaskId.java

Co-Authored-By: Hao Chen <chenh1024@gmail.com>

* Update java/api/src/main/java/org/ray/api/id/TaskId.java

Co-Authored-By: Hao Chen <chenh1024@gmail.com>

* Update java/api/src/main/java/org/ray/api/id/ActorId.java

Co-Authored-By: Hao Chen <chenh1024@gmail.com>

* Address comments

* Add DriverTaskId embeding job id

* Fix tests

* Add python dor_fake_driver_id

* Address comments and fix linting

* Fix CI
This commit is contained in:
Qing Wang
2019-08-07 11:04:51 +08:00
committed by GitHub
parent 50b93bf179
commit d372f24e3c
71 changed files with 1368 additions and 586 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/test/src/main/java/org/ray/api/test/ActorReconstructionTest.java
+4 -3
View File
@@ -11,6 +11,7 @@ import org.ray.api.RayActor;
import org.ray.api.TestUtils;
import org.ray.api.annotation.RayRemote;
import org.ray.api.exception.RayActorException;
import org.ray.api.id.ActorId;
import org.ray.api.id.UniqueId;
import org.ray.api.options.ActorCreationOptions;
import org.testng.Assert;
@@ -106,13 +107,13 @@ public class ActorReconstructionTest extends BaseTest {
}
@Override
public void saveCheckpoint(UniqueId actorId, UniqueId checkpointId) {
public void saveCheckpoint(ActorId actorId, UniqueId checkpointId) {
// In practice, user should save the checkpoint id and data to a persistent store.
// But for simplicity, we don't do that in this unit test.
}
@Override
public UniqueId loadCheckpoint(UniqueId actorId, List<Checkpoint> availableCheckpoints) {
public UniqueId loadCheckpoint(ActorId actorId, List<Checkpoint> availableCheckpoints) {
// Restore previous value and return checkpoint id.
this.value = 3;
this.resumedFromCheckpoint = true;
@@ -120,7 +121,7 @@ public class ActorReconstructionTest extends BaseTest {
}
@Override
public void checkpointExpired(UniqueId actorId, UniqueId checkpointId) {
public void checkpointExpired(ActorId actorId, UniqueId checkpointId) {
}
}
java/test/src/main/java/org/ray/api/test/ClientExceptionTest.java
+1 -1
View File
@@ -20,7 +20,7 @@ public class ClientExceptionTest extends BaseTest {
@Test
public void testWaitAndCrash() {
TestUtils.skipTestUnderSingleProcess();
ObjectId randomId = ObjectId.randomId();
ObjectId randomId = ObjectId.fromRandom();
RayObject<String> notExisting = new RayObjectImpl(randomId);
Thread thread = new Thread(() -> {
java/test/src/main/java/org/ray/api/test/PlasmaStoreTest.java
+1 -1
View File
@@ -13,7 +13,7 @@ public class PlasmaStoreTest extends BaseTest {
@Test
public void testPutWithDuplicateId() {
TestUtils.skipTestUnderSingleProcess();
ObjectId objectId = ObjectId.randomId();
ObjectId objectId = ObjectId.fromRandom();
AbstractRayRuntime runtime = (AbstractRayRuntime) Ray.internal();
ObjectStoreProxy objectInterface = runtime.getObjectStoreProxy();
objectInterface.put(objectId, 1);
java/test/src/main/java/org/ray/api/test/RaySerializerTest.java
+3 -1
View File
@@ -1,6 +1,8 @@
package org.ray.api.test;
import org.ray.api.RayPyActor;
import org.ray.api.id.ActorId;
import org.ray.api.id.JobId;
import org.ray.api.id.UniqueId;
import org.ray.runtime.RayPyActorImpl;
import org.ray.runtime.util.Serializer;
@@ -11,7 +13,7 @@ public class RaySerializerTest {
@Test
public void testSerializePyActor() {
final UniqueId pyActorId = UniqueId.randomId();
final ActorId pyActorId = ActorId.generateActorId(JobId.fromInt(1));
RayPyActor pyActor = new RayPyActorImpl(pyActorId, "test", "RaySerializerTest");
byte[] bytes = Serializer.encode(pyActor);
RayPyActor result = Serializer.decode(bytes);
java/test/src/main/java/org/ray/api/test/RuntimeContextTest.java
+2 -1
View File
@@ -3,6 +3,7 @@ package org.ray.api.test;
import org.ray.api.Ray;
import org.ray.api.RayActor;
import org.ray.api.annotation.RayRemote;
import org.ray.api.id.ActorId;
import org.ray.api.id.JobId;
import org.ray.api.id.UniqueId;
import org.testng.Assert;
@@ -41,7 +42,7 @@ public class RuntimeContextTest extends BaseTest {
@RayRemote
public static class RuntimeContextTester {
public String testRuntimeContext(UniqueId actorId) {
public String testRuntimeContext(ActorId actorId) {
Assert.assertEquals(JOB_ID, Ray.getRuntimeContext().getCurrentJobId());
Assert.assertEquals(actorId, Ray.getRuntimeContext().getCurrentActorId());
Assert.assertEquals(RAYLET_SOCKET_NAME, Ray.getRuntimeContext().getRayletSocketName());
java/test/src/main/java/org/ray/api/test/UniqueIdTest.java
+12 -54
View File
@@ -3,6 +3,7 @@ package org.ray.api.test;
import java.nio.ByteBuffer;
import java.util.Arrays;
import javax.xml.bind.DatatypeConverter;
import org.ray.api.id.ObjectId;
import org.ray.api.id.TaskId;
import org.ray.api.id.UniqueId;
@@ -52,49 +53,26 @@ public class UniqueIdTest {
@Test
public void testComputeReturnId() {
// Mock a taskId, and the lowest 4 bytes should be 0.
TaskId taskId = TaskId.fromHexString("123456789ABCDEF123456789ABCDEF00");
TaskId taskId = TaskId.fromHexString("123456789ABCDE123456789ABCDE");
ObjectId returnId = IdUtil.computeReturnId(taskId, 1);
Assert.assertEquals("123456789abcdef123456789abcdef0001000000", returnId.toString());
ObjectId returnId = ObjectId.forReturn(taskId, 1);
Assert.assertEquals("123456789abcde123456789abcde00c001000000", returnId.toString());
Assert.assertEquals(returnId.getTaskId(), taskId);
returnId = IdUtil.computeReturnId(taskId, 0x01020304);
Assert.assertEquals("123456789abcdef123456789abcdef0004030201", returnId.toString());
}
@Test
public void testComputeTaskId() {
ObjectId objId = ObjectId.fromHexString("123456789ABCDEF123456789ABCDEF0034421980");
TaskId taskId = objId.getTaskId();
Assert.assertEquals("123456789abcdef123456789abcdef00", taskId.toString());
returnId = ObjectId.forReturn(taskId, 0x01020304);
Assert.assertEquals("123456789abcde123456789abcde00c004030201", returnId.toString());
}
@Test
public void testComputePutId() {
// Mock a taskId, the lowest 4 bytes should be 0.
TaskId taskId = TaskId.fromHexString("123456789ABCDEF123456789ABCDEF00");
TaskId taskId = TaskId.fromHexString("123456789ABCDE123456789ABCDE");
ObjectId putId = IdUtil.computePutId(taskId, 1);
Assert.assertEquals("123456789ABCDEF123456789ABCDEF00FFFFFFFF".toLowerCase(), putId.toString());
ObjectId putId = ObjectId.forPut(taskId, 1);
Assert.assertEquals("123456789abcde123456789abcde008001000000".toLowerCase(), putId.toString());
putId = IdUtil.computePutId(taskId, 0x01020304);
Assert.assertEquals("123456789ABCDEF123456789ABCDEF00FCFCFDFE".toLowerCase(), putId.toString());
}
@Test
public void testUniqueIdsAndByteBufferInterConversion() {
final int len = 5;
UniqueId[] ids = new UniqueId[len];
for (int i = 0; i < len; ++i) {
ids[i] = UniqueId.randomId();
}
ByteBuffer temp = IdUtil.concatIds(ids);
UniqueId[] res = IdUtil.getUniqueIdsFromByteBuffer(temp);
for (int i = 0; i < len; ++i) {
Assert.assertEquals(ids[i], res[i]);
}
putId = ObjectId.forPut(taskId, 0x01020304);
Assert.assertEquals("123456789abcde123456789abcde008004030201".toLowerCase(), putId.toString());
}
@Test
@@ -104,24 +82,4 @@ public class UniqueIdTest {
Assert.assertEquals(remainder, 787616861);
}
@Test
void testConcateIds() {
String taskHexStr = "123456789ABCDEF123456789ABCDEF00";
String objectHexStr = taskHexStr + "01020304";
ObjectId objectId1 = ObjectId.fromHexString(objectHexStr);
ObjectId objectId2 = ObjectId.fromHexString(objectHexStr);
TaskId[] taskIds = new TaskId[2];
taskIds[0] = objectId1.getTaskId();
taskIds[1] = objectId2.getTaskId();
ObjectId[] objectIds = new ObjectId[2];
objectIds[0] = objectId1;
objectIds[1] = objectId2;
String taskHexCompareStr = taskHexStr + taskHexStr;
String objectHexCompareStr = objectHexStr + objectHexStr;
Assert.assertEquals(DatatypeConverter.printHexBinary(
IdUtil.concatIds(taskIds).array()), taskHexCompareStr);
Assert.assertEquals(DatatypeConverter.printHexBinary(
IdUtil.concatIds(objectIds).array()), objectHexCompareStr);
}
}