[multi-language part 1] add a 'language' field to task specification (#2639)

java/checkstyle-suppressions.xml

@@ -3,21 +3,19 @@
 "http://www.puppycrawl.com/dtds/suppressions_1_1.dtd">
 
 <suppressions>
-  <suppress checks="ParameterNameCheck" files="Arg.java"/>
-  <suppress checks="MethodNameCheck" files="Arg.java"/>
   <suppress checks="MethodNameCheck" files="DefaultLocalSchedulerClient.java"/>
   <suppress checks="MemberNameCheck" files="PathConfig.java"/>
   <suppress checks="MemberNameCheck" files="RayParameters.java"/>
   <suppress checks="AbbreviationAsWordInNameCheck" files="RayParameters.java"/>
   <suppress checks="MethodNameCheck" files="ResourcePair.java"/>
   <suppress checks="ParameterNameCheck" files="ResourcePair.java"/>
-  <suppress checks="ParameterNameCheck" files="TaskInfo.java"/>
-  <suppress checks="MethodNameCheck" files="TaskInfo.java"/>
   <suppress checks="MethodNameCheck" files="RayMap.java"/>
   <suppress checks="MethodNameCheck" files="RayList.java"/>
   <suppress checks="OverloadMethodsDeclarationOrderCheck" files="Rpc.java"/>
   <suppress checks="MethodTypeParameterNameCheck" files="Rpc.java"/>
   <suppress checks="AbbreviationAsWordInNameCheck" files="UniqueID.java"/>
   <suppress checks="TypeNameCheck" files="RayFunc_[0-9]_.+\.java"/>
-  <suppress checks=".*" files="[/\\]format[/\\]" />
+  <!-- suppress check for flatbuffer-generated files. -->
+  <!-- TODO(raulchen): move these files to a directory, so this rule can be simplier. -->
+  <suppress checks=".*" files="(Arg|ResourcePair|TaskLanguage|TaskInfo|ClientTableData).java" />
 </suppressions>

java/pom.xml

@@ -130,13 +130,6 @@
                 <version>${slf4j.version}</version>
             </dependency>
 
-            <!-- https://mvnrepository.com/artifact/com.github.davidmoten/flatbuffers-java -->
-            <dependency>
-                <groupId>com.github.davidmoten</groupId>
-                <artifactId>flatbuffers-java</artifactId>
-                <version>1.7.0.1</version>
-            </dependency>
-
             <!-- https://mvnrepository.com/artifact/com.beust/jcommander -->
             <dependency>
                 <groupId>com.beust</groupId>

java/runtime-native/src/main/java/org/ray/spi/impl/Arg.java

@@ -1,90 +1,44 @@
-package org.ray.spi.impl;
-
 // automatically generated by the FlatBuffers compiler, do not modify
 
-import com.google.flatbuffers.FlatBufferBuilder;
-import com.google.flatbuffers.Table;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
+package org.ray.spi.impl;
+
+import java.nio.*;
+import java.lang.*;
+import java.util.*;
+import com.google.flatbuffers.*;
 
 @SuppressWarnings("unused")
 public final class Arg extends Table {
-  public static Arg getRootAsArg(ByteBuffer _bb) {
-    return getRootAsArg(_bb, new Arg());
-  }
+  public static Arg getRootAsArg(ByteBuffer _bb) { return getRootAsArg(_bb, new Arg()); }
+  public static Arg getRootAsArg(ByteBuffer _bb, Arg obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); }
+  public void __init(int _i, ByteBuffer _bb) { bb_pos = _i; bb = _bb; }
+  public Arg __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; }
 
-  public static Arg getRootAsArg(ByteBuffer _bb, Arg obj) {
-    _bb.order(ByteOrder.LITTLE_ENDIAN);
-    return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb));
-  }
-
-  public Arg __assign(int _i, ByteBuffer _bb) {
-    __init(_i, _bb);
-    return this;
-  }
-
-  public void __init(int _i, ByteBuffer _bb) {
-    bb_pos = _i;
-    bb = _bb;
-  }
+  public String objectIds(int j) { int o = __offset(4); return o != 0 ? __string(__vector(o) + j * 4) : null; }
+  public int objectIdsLength() { int o = __offset(4); return o != 0 ? __vector_len(o) : 0; }
+  public String data() { int o = __offset(6); return o != 0 ? __string(o + bb_pos) : null; }
+  public ByteBuffer dataAsByteBuffer() { return __vector_as_bytebuffer(6, 1); }
+  public ByteBuffer dataInByteBuffer(ByteBuffer _bb) { return __vector_in_bytebuffer(_bb, 6, 1); }
 
   public static int createArg(FlatBufferBuilder builder,
-                              int object_idsOffset,
-                              int dataOffset) {
+      int object_idsOffset,
+      int dataOffset) {
     builder.startObject(2);
     Arg.addData(builder, dataOffset);
     Arg.addObjectIds(builder, object_idsOffset);
     return Arg.endArg(builder);
   }
 
-  public static void addData(FlatBufferBuilder builder, int dataOffset) {
-    builder.addOffset(1, dataOffset, 0);
-  }
-
-  public static void addObjectIds(FlatBufferBuilder builder, int objectIdsOffset) {
-    builder.addOffset(0, objectIdsOffset, 0);
-  }
-
+  public static void startArg(FlatBufferBuilder builder) { builder.startObject(2); }
+  public static void addObjectIds(FlatBufferBuilder builder, int objectIdsOffset) { builder.addOffset(0, objectIdsOffset, 0); }
+  public static int createObjectIdsVector(FlatBufferBuilder builder, int[] data) { builder.startVector(4, data.length, 4); for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); return builder.endVector(); }
+  public static void startObjectIdsVector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems, 4); }
+  public static void addData(FlatBufferBuilder builder, int dataOffset) { builder.addOffset(1, dataOffset, 0); }
   public static int endArg(FlatBufferBuilder builder) {
     int o = builder.endObject();
     return o;
   }
 
-  public static void startArg(FlatBufferBuilder builder) {
-    builder.startObject(2);
-  }
-
-  public static int createObjectIdsVector(FlatBufferBuilder builder, int[] data) {
-    builder.startVector(4, data.length, 4);
-    for (int i = data.length - 1; i >= 0; i--) {
-      builder.addOffset(data[i]);
-    }
-    return builder.endVector();
-  }
-
-  public static void startObjectIdsVector(FlatBufferBuilder builder, int numElems) {
-    builder.startVector(4, numElems, 4);
-  }
-
-  public String objectIds(int j) {
-    int o = __offset(4);
-    return o != 0 ? __string(__vector(o) + j * 4) : null;
-  }
-
-  public int objectIdsLength() {
-    int o = __offset(4);
-    return o != 0 ? __vector_len(o) : 0;
-  }
-
-  public String data() {
-    int o = __offset(6);
-    return o != 0 ? __string(o + bb_pos) : null;
-  }
-
-  public ByteBuffer dataAsByteBuffer() {
-    return __vector_as_bytebuffer(6, 1);
-  }
-
   //this is manually added to avoid encoding/decoding cost as our object id is a byte array
   // instead of a string
   public ByteBuffer objectIdAsByteBuffer(int j) {
@@ -103,4 +57,3 @@ public final class Arg extends Table {
   }
 }
 
-

java/runtime-native/src/main/java/org/ray/spi/impl/DefaultLocalSchedulerClient.java

@@ -253,7 +253,7 @@ public class DefaultLocalSchedulerClient implements LocalSchedulerLink {
         actorCreateIdOffset, actorCreateDummyIdOffset,
         actorIdOffset, actorHandleIdOffset, actorCounter,
         false, functionIdOffset,
-        argsOffset, returnsOffset, requiredResourcesOffset);
+        argsOffset, returnsOffset, requiredResourcesOffset, TaskLanguage.JAVA);
 
     fbb.finish(root);
     ByteBuffer buffer = fbb.dataBuffer();

java/runtime-native/src/main/java/org/ray/spi/impl/ResourcePair.java

@@ -1,71 +1,39 @@
-package org.ray.spi.impl;
-
 // automatically generated by the FlatBuffers compiler, do not modify
 
-import com.google.flatbuffers.FlatBufferBuilder;
-import com.google.flatbuffers.Table;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
+package org.ray.spi.impl;
+
+import java.nio.*;
+import java.lang.*;
+import java.util.*;
+import com.google.flatbuffers.*;
 
 @SuppressWarnings("unused")
 public final class ResourcePair extends Table {
-  public static ResourcePair getRootAsResourcePair(ByteBuffer _bb) {
-    return getRootAsResourcePair(_bb, new ResourcePair());
-  }
+  public static ResourcePair getRootAsResourcePair(ByteBuffer _bb) { return getRootAsResourcePair(_bb, new ResourcePair()); }
+  public static ResourcePair getRootAsResourcePair(ByteBuffer _bb, ResourcePair obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); }
+  public void __init(int _i, ByteBuffer _bb) { bb_pos = _i; bb = _bb; }
+  public ResourcePair __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; }
 
-  public static ResourcePair getRootAsResourcePair(ByteBuffer _bb, ResourcePair obj) {
-    _bb.order(ByteOrder.LITTLE_ENDIAN);
-    return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb));
-  }
-
-  public ResourcePair __assign(int _i, ByteBuffer _bb) {
-    __init(_i, _bb);
-    return this;
-  }
-
-  public void __init(int _i, ByteBuffer _bb) {
-    bb_pos = _i;
-    bb = _bb;
-  }
+  public String key() { int o = __offset(4); return o != 0 ? __string(o + bb_pos) : null; }
+  public ByteBuffer keyAsByteBuffer() { return __vector_as_bytebuffer(4, 1); }
+  public ByteBuffer keyInByteBuffer(ByteBuffer _bb) { return __vector_in_bytebuffer(_bb, 4, 1); }
+  public double value() { int o = __offset(6); return o != 0 ? bb.getDouble(o + bb_pos) : 0.0; }
 
   public static int createResourcePair(FlatBufferBuilder builder,
-                                       int keyOffset,
-                                       double value) {
+      int keyOffset,
+      double value) {
     builder.startObject(2);
     ResourcePair.addValue(builder, value);
     ResourcePair.addKey(builder, keyOffset);
     return ResourcePair.endResourcePair(builder);
   }
 
-  public static void addValue(FlatBufferBuilder builder, double value) {
-    builder.addDouble(1, value, 0.0);
-  }
-
-  public static void addKey(FlatBufferBuilder builder, int keyOffset) {
-    builder.addOffset(0, keyOffset, 0);
-  }
-
+  public static void startResourcePair(FlatBufferBuilder builder) { builder.startObject(2); }
+  public static void addKey(FlatBufferBuilder builder, int keyOffset) { builder.addOffset(0, keyOffset, 0); }
+  public static void addValue(FlatBufferBuilder builder, double value) { builder.addDouble(1, value, 0.0); }
   public static int endResourcePair(FlatBufferBuilder builder) {
     int o = builder.endObject();
     return o;
   }
-
-  public static void startResourcePair(FlatBufferBuilder builder) {
-    builder.startObject(2);
-  }
-
-  public String key() {
-    int o = __offset(4);
-    return o != 0 ? __string(o + bb_pos) : null;
-  }
-
-  public ByteBuffer keyAsByteBuffer() {
-    return __vector_as_bytebuffer(4, 1);
-  }
-
-  public double value() {
-    int o = __offset(6);
-    return o != 0 ? bb.getDouble(o + bb_pos) : 0.0;
-  }
 }
 

java/runtime-native/src/main/java/org/ray/spi/impl/TaskInfo.java

@@ -1,48 +1,74 @@
-package org.ray.spi.impl;
 // automatically generated by the FlatBuffers compiler, do not modify
 
-import com.google.flatbuffers.FlatBufferBuilder;
-import com.google.flatbuffers.Table;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
+package org.ray.spi.impl;
+
+import java.nio.*;
+import java.lang.*;
+import java.util.*;
+import com.google.flatbuffers.*;
 
 @SuppressWarnings("unused")
 public final class TaskInfo extends Table {
-  public static TaskInfo getRootAsTaskInfo(ByteBuffer _bb) {
-    return getRootAsTaskInfo(_bb, new TaskInfo());
-  }
+  public static TaskInfo getRootAsTaskInfo(ByteBuffer _bb) { return getRootAsTaskInfo(_bb, new TaskInfo()); }
+  public static TaskInfo getRootAsTaskInfo(ByteBuffer _bb, TaskInfo obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); }
+  public void __init(int _i, ByteBuffer _bb) { bb_pos = _i; bb = _bb; }
+  public TaskInfo __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; }
 
-  public static TaskInfo getRootAsTaskInfo(ByteBuffer _bb, TaskInfo obj) {
-    _bb.order(ByteOrder.LITTLE_ENDIAN);
-    return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb));
-  }
-
-  public TaskInfo __assign(int _i, ByteBuffer _bb) {
-    __init(_i, _bb);
-    return this;
-  }
-
-  public void __init(int _i, ByteBuffer _bb) {
-    bb_pos = _i;
-    bb = _bb;
-  }
+  public String driverId() { int o = __offset(4); return o != 0 ? __string(o + bb_pos) : null; }
+  public ByteBuffer driverIdAsByteBuffer() { return __vector_as_bytebuffer(4, 1); }
+  public ByteBuffer driverIdInByteBuffer(ByteBuffer _bb) { return __vector_in_bytebuffer(_bb, 4, 1); }
+  public String taskId() { int o = __offset(6); return o != 0 ? __string(o + bb_pos) : null; }
+  public ByteBuffer taskIdAsByteBuffer() { return __vector_as_bytebuffer(6, 1); }
+  public ByteBuffer taskIdInByteBuffer(ByteBuffer _bb) { return __vector_in_bytebuffer(_bb, 6, 1); }
+  public String parentTaskId() { int o = __offset(8); return o != 0 ? __string(o + bb_pos) : null; }
+  public ByteBuffer parentTaskIdAsByteBuffer() { return __vector_as_bytebuffer(8, 1); }
+  public ByteBuffer parentTaskIdInByteBuffer(ByteBuffer _bb) { return __vector_in_bytebuffer(_bb, 8, 1); }
+  public int parentCounter() { int o = __offset(10); return o != 0 ? bb.getInt(o + bb_pos) : 0; }
+  public String actorCreationId() { int o = __offset(12); return o != 0 ? __string(o + bb_pos) : null; }
+  public ByteBuffer actorCreationIdAsByteBuffer() { return __vector_as_bytebuffer(12, 1); }
+  public ByteBuffer actorCreationIdInByteBuffer(ByteBuffer _bb) { return __vector_in_bytebuffer(_bb, 12, 1); }
+  public String actorCreationDummyObjectId() { int o = __offset(14); return o != 0 ? __string(o + bb_pos) : null; }
+  public ByteBuffer actorCreationDummyObjectIdAsByteBuffer() { return __vector_as_bytebuffer(14, 1); }
+  public ByteBuffer actorCreationDummyObjectIdInByteBuffer(ByteBuffer _bb) { return __vector_in_bytebuffer(_bb, 14, 1); }
+  public String actorId() { int o = __offset(16); return o != 0 ? __string(o + bb_pos) : null; }
+  public ByteBuffer actorIdAsByteBuffer() { return __vector_as_bytebuffer(16, 1); }
+  public ByteBuffer actorIdInByteBuffer(ByteBuffer _bb) { return __vector_in_bytebuffer(_bb, 16, 1); }
+  public String actorHandleId() { int o = __offset(18); return o != 0 ? __string(o + bb_pos) : null; }
+  public ByteBuffer actorHandleIdAsByteBuffer() { return __vector_as_bytebuffer(18, 1); }
+  public ByteBuffer actorHandleIdInByteBuffer(ByteBuffer _bb) { return __vector_in_bytebuffer(_bb, 18, 1); }
+  public int actorCounter() { int o = __offset(20); return o != 0 ? bb.getInt(o + bb_pos) : 0; }
+  public boolean isActorCheckpointMethod() { int o = __offset(22); return o != 0 ? 0!=bb.get(o + bb_pos) : false; }
+  public String functionId() { int o = __offset(24); return o != 0 ? __string(o + bb_pos) : null; }
+  public ByteBuffer functionIdAsByteBuffer() { return __vector_as_bytebuffer(24, 1); }
+  public ByteBuffer functionIdInByteBuffer(ByteBuffer _bb) { return __vector_in_bytebuffer(_bb, 24, 1); }
+  public Arg args(int j) { return args(new Arg(), j); }
+  public Arg args(Arg obj, int j) { int o = __offset(26); return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null; }
+  public int argsLength() { int o = __offset(26); return o != 0 ? __vector_len(o) : 0; }
+  public String returns(int j) { int o = __offset(28); return o != 0 ? __string(__vector(o) + j * 4) : null; }
+  public int returnsLength() { int o = __offset(28); return o != 0 ? __vector_len(o) : 0; }
+  public ResourcePair requiredResources(int j) { return requiredResources(new ResourcePair(), j); }
+  public ResourcePair requiredResources(ResourcePair obj, int j) { int o = __offset(30); return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null; }
+  public int requiredResourcesLength() { int o = __offset(30); return o != 0 ? __vector_len(o) : 0; }
+  public int language() { int o = __offset(32); return o != 0 ? bb.getInt(o + bb_pos) : 0; }
 
   public static int createTaskInfo(FlatBufferBuilder builder,
-                                   int driver_idOffset,
-                                   int task_idOffset,
-                                   int parent_task_idOffset,
-                                   int parent_counter,
-                                   int actor_creation_idOffset,
-                                   int actor_creation_dummy_object_idOffset,
-                                   int actor_idOffset,
-                                   int actor_handle_idOffset,
-                                   int actor_counter,
-                                   boolean is_actor_checkpoint_method,
-                                   int function_idOffset,
-                                   int argsOffset,
-                                   int returnsOffset,
-                                   int required_resourcesOffset) {
-    builder.startObject(14);
+      int driver_idOffset,
+      int task_idOffset,
+      int parent_task_idOffset,
+      int parent_counter,
+      int actor_creation_idOffset,
+      int actor_creation_dummy_object_idOffset,
+      int actor_idOffset,
+      int actor_handle_idOffset,
+      int actor_counter,
+      boolean is_actor_checkpoint_method,
+      int function_idOffset,
+      int argsOffset,
+      int returnsOffset,
+      int required_resourcesOffset,
+      int language) {
+    builder.startObject(15);
+    TaskInfo.addLanguage(builder, language);
     TaskInfo.addRequiredResources(builder, required_resourcesOffset);
     TaskInfo.addReturns(builder, returnsOffset);
     TaskInfo.addArgs(builder, argsOffset);
@@ -60,234 +86,33 @@ public final class TaskInfo extends Table {
     return TaskInfo.endTaskInfo(builder);
   }
 
-  public static void addRequiredResources(FlatBufferBuilder builder, int requiredResourcesOffset) {
-    builder.addOffset(13, requiredResourcesOffset, 0);
-  }
-
-  public static void addReturns(FlatBufferBuilder builder, int returnsOffset) {
-    builder.addOffset(12, returnsOffset, 0);
-  }
-
-  public static void addArgs(FlatBufferBuilder builder, int argsOffset) {
-    builder.addOffset(11, argsOffset, 0);
-  }
-
-  public static void addFunctionId(FlatBufferBuilder builder, int functionIdOffset) {
-    builder.addOffset(10, functionIdOffset, 0);
-  }
-
-  public static void addActorCounter(FlatBufferBuilder builder, int actorCounter) {
-    builder.addInt(8, actorCounter, 0);
-  }
-
-  public static void addActorHandleId(FlatBufferBuilder builder, int actorHandleIdOffset) {
-    builder.addOffset(7, actorHandleIdOffset, 0);
-  }
-
-  public static void addActorId(FlatBufferBuilder builder, int actorIdOffset) {
-    builder.addOffset(6, actorIdOffset, 0);
-  }
-
-  public static void addActorCreationDummyObjectId(FlatBufferBuilder builder, int
-      actorCreationDummyObjectIdOffset) {
-    builder.addOffset(5, actorCreationDummyObjectIdOffset, 0);
-  }
-
-  public static void addActorCreationId(FlatBufferBuilder builder, int actorCreationIdOffset) {
-    builder.addOffset(4, actorCreationIdOffset, 0);
-  }
-
-  public static void addParentCounter(FlatBufferBuilder builder, int parentCounter) {
-    builder.addInt(3, parentCounter, 0);
-  }
-
-  public static void addParentTaskId(FlatBufferBuilder builder, int parentTaskIdOffset) {
-    builder.addOffset(2, parentTaskIdOffset, 0);
-  }
-
-  public static void addTaskId(FlatBufferBuilder builder, int taskIdOffset) {
-    builder.addOffset(1, taskIdOffset, 0);
-  }
-
-  public static void addDriverId(FlatBufferBuilder builder, int driverIdOffset) {
-    builder.addOffset(0, driverIdOffset, 0);
-  }
-
-  public static void addIsActorCheckpointMethod(FlatBufferBuilder builder, boolean
-      isActorCheckpointMethod) {
-    builder.addBoolean(9, isActorCheckpointMethod, false);
-  }
-
+  public static void startTaskInfo(FlatBufferBuilder builder) { builder.startObject(15); }
+  public static void addDriverId(FlatBufferBuilder builder, int driverIdOffset) { builder.addOffset(0, driverIdOffset, 0); }
+  public static void addTaskId(FlatBufferBuilder builder, int taskIdOffset) { builder.addOffset(1, taskIdOffset, 0); }
+  public static void addParentTaskId(FlatBufferBuilder builder, int parentTaskIdOffset) { builder.addOffset(2, parentTaskIdOffset, 0); }
+  public static void addParentCounter(FlatBufferBuilder builder, int parentCounter) { builder.addInt(3, parentCounter, 0); }
+  public static void addActorCreationId(FlatBufferBuilder builder, int actorCreationIdOffset) { builder.addOffset(4, actorCreationIdOffset, 0); }
+  public static void addActorCreationDummyObjectId(FlatBufferBuilder builder, int actorCreationDummyObjectIdOffset) { builder.addOffset(5, actorCreationDummyObjectIdOffset, 0); }
+  public static void addActorId(FlatBufferBuilder builder, int actorIdOffset) { builder.addOffset(6, actorIdOffset, 0); }
+  public static void addActorHandleId(FlatBufferBuilder builder, int actorHandleIdOffset) { builder.addOffset(7, actorHandleIdOffset, 0); }
+  public static void addActorCounter(FlatBufferBuilder builder, int actorCounter) { builder.addInt(8, actorCounter, 0); }
+  public static void addIsActorCheckpointMethod(FlatBufferBuilder builder, boolean isActorCheckpointMethod) { builder.addBoolean(9, isActorCheckpointMethod, false); }
+  public static void addFunctionId(FlatBufferBuilder builder, int functionIdOffset) { builder.addOffset(10, functionIdOffset, 0); }
+  public static void addArgs(FlatBufferBuilder builder, int argsOffset) { builder.addOffset(11, argsOffset, 0); }
+  public static int createArgsVector(FlatBufferBuilder builder, int[] data) { builder.startVector(4, data.length, 4); for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); return builder.endVector(); }
+  public static void startArgsVector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems, 4); }
+  public static void addReturns(FlatBufferBuilder builder, int returnsOffset) { builder.addOffset(12, returnsOffset, 0); }
+  public static int createReturnsVector(FlatBufferBuilder builder, int[] data) { builder.startVector(4, data.length, 4); for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); return builder.endVector(); }
+  public static void startReturnsVector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems, 4); }
+  public static void addRequiredResources(FlatBufferBuilder builder, int requiredResourcesOffset) { builder.addOffset(13, requiredResourcesOffset, 0); }
+  public static int createRequiredResourcesVector(FlatBufferBuilder builder, int[] data) { builder.startVector(4, data.length, 4); for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); return builder.endVector(); }
+  public static void startRequiredResourcesVector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems, 4); }
+  public static void addLanguage(FlatBufferBuilder builder, int language) { builder.addInt(14, language, 0); }
   public static int endTaskInfo(FlatBufferBuilder builder) {
     int o = builder.endObject();
     return o;
   }
 
-  public static void startTaskInfo(FlatBufferBuilder builder) {
-    builder.startObject(14);
-  }
-
-  public static int createArgsVector(FlatBufferBuilder builder, int[] data) {
-    builder.startVector(4, data.length, 4);
-    for (int i = data.length - 1; i >= 0; i--) {
-      builder.addOffset(data[i]);
-    }
-    return builder.endVector();
-  }
-
-  public static void startArgsVector(FlatBufferBuilder builder, int numElems) {
-    builder.startVector(4, numElems, 4);
-  }
-
-  public static int createReturnsVector(FlatBufferBuilder builder, int[] data) {
-    builder.startVector(4, data.length, 4);
-    for (int i = data.length - 1; i >= 0; i--) {
-      builder.addOffset(data[i]);
-    }
-    return builder.endVector();
-  }
-
-  public static void startReturnsVector(FlatBufferBuilder builder, int numElems) {
-    builder.startVector(4, numElems, 4);
-  }
-
-  public static int createRequiredResourcesVector(FlatBufferBuilder builder, int[] data) {
-    builder.startVector(4, data.length, 4);
-    for (int i = data.length - 1; i >= 0; i--) {
-      builder.addOffset(data[i]);
-    }
-    return builder.endVector();
-  }
-
-  public static void startRequiredResourcesVector(FlatBufferBuilder builder, int numElems) {
-    builder.startVector(4, numElems, 4);
-  }
-
-  public String driverId() {
-    int o = __offset(4);
-    return o != 0 ? __string(o + bb_pos) : null;
-  }
-
-  public ByteBuffer driverIdAsByteBuffer() {
-    return __vector_as_bytebuffer(4, 1);
-  }
-
-  public String taskId() {
-    int o = __offset(6);
-    return o != 0 ? __string(o + bb_pos) : null;
-  }
-
-  public ByteBuffer taskIdAsByteBuffer() {
-    return __vector_as_bytebuffer(6, 1);
-  }
-
-  public String parentTaskId() {
-    int o = __offset(8);
-    return o != 0 ? __string(o + bb_pos) : null;
-  }
-
-  public ByteBuffer parentTaskIdAsByteBuffer() {
-    return __vector_as_bytebuffer(8, 1);
-  }
-
-  public int parentCounter() {
-    int o = __offset(10);
-    return o != 0 ? bb.getInt(o + bb_pos) : 0;
-  }
-
-  public String actorCreationId() {
-    int o = __offset(12);
-    return o != 0 ? __string(o + bb_pos) : null;
-  }
-
-  public ByteBuffer actorCreationIdAsByteBuffer() {
-    return __vector_as_bytebuffer(12, 1);
-  }
-
-  public String actorCreationDummyObjectId() {
-    int o = __offset(14);
-    return o != 0 ? __string(o + bb_pos) : null;
-  }
-
-  public ByteBuffer actorCreationDummyObjectIdAsByteBuffer() {
-    return __vector_as_bytebuffer(14, 1);
-  }
-
-  public String actorId() {
-    int o = __offset(16);
-    return o != 0 ? __string(o + bb_pos) : null;
-  }
-
-  public ByteBuffer actorIdAsByteBuffer() {
-    return __vector_as_bytebuffer(16, 1);
-  }
-
-  public String actorHandleId() {
-    int o = __offset(18);
-    return o != 0 ? __string(o + bb_pos) : null;
-  }
-
-  public ByteBuffer actorHandleIdAsByteBuffer() {
-    return __vector_as_bytebuffer(18, 1);
-  }
-
-  public int actorCounter() {
-    int o = __offset(20);
-    return o != 0 ? bb.getInt(o + bb_pos) : 0;
-  }
-
-  public boolean isActorCheckpointMethod() {
-    int o = __offset(22);
-    return o != 0 ? 0 != bb.get(o + bb_pos) : false;
-  }
-
-  public String functionId() {
-    int o = __offset(24);
-    return o != 0 ? __string(o + bb_pos) : null;
-  }
-
-  public ByteBuffer functionIdAsByteBuffer() {
-    return __vector_as_bytebuffer(24, 1);
-  }
-
-  public Arg args(int j) {
-    return args(new Arg(), j);
-  }
-
-  public Arg args(Arg obj, int j) {
-    int o = __offset(26);
-    return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null;
-  }
-
-  public int argsLength() {
-    int o = __offset(26);
-    return o != 0 ? __vector_len(o) : 0;
-  }
-
-  public String returns(int j) {
-    int o = __offset(28);
-    return o != 0 ? __string(__vector(o) + j * 4) : null;
-  }
-
-  public int returnsLength() {
-    int o = __offset(28);
-    return o != 0 ? __vector_len(o) : 0;
-  }
-
-  public ResourcePair requiredResources(int j) {
-    return requiredResources(new ResourcePair(), j);
-  }
-
-  public ResourcePair requiredResources(ResourcePair obj, int j) {
-    int o = __offset(30);
-    return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null;
-  }
-
-  public int requiredResourcesLength() {
-    int o = __offset(30);
-    return o != 0 ? __vector_len(o) : 0;
-  }
-
   //this is manually added to avoid encoding/decoding cost as our object
   //id is a byte array instead of a string
   public ByteBuffer returnsAsByteBuffer(int j) {

java/runtime-native/src/main/java/org/ray/spi/impl/TaskLanguage.java

@@ -0,0 +1,14 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+package org.ray.spi.impl;
+
+public final class TaskLanguage {
+  private TaskLanguage() { }
+  public static final int PYTHON = 0;
+  public static final int JAVA = 1;
+
+  public static final String[] names = { "PYTHON", "JAVA", };
+
+  public static String name(int e) { return names[e]; }
+}
+

src/common/format/common.fbs

@@ -20,6 +20,14 @@ table ResourcePair {
   value: double;
 }
 
+// NOTE: This enum is duplicate with the `Language` enum in `gcs.fbs`,
+// because we cannot include this file in `gcs.fbs` due to cyclic dependency.
+// TODO(raulchen): remove it once we get rid of legacy ray.
+enum TaskLanguage:int {
+  PYTHON = 0,
+  JAVA = 1
+}
+
 table TaskInfo {
   // ID of the driver that created this task.
   driver_id: string;
@@ -52,6 +60,8 @@ table TaskInfo {
   // The required_resources vector indicates the quantities of the different
   // resources required by this task.
   required_resources: [ResourcePair];
+  // The language that this task belongs to
+  language: TaskLanguage;
 }
 
 // Object information data structure.

src/common/lib/python/common_extension.cc

@@ -462,7 +462,8 @@ static int PyTask_init(PyTask *self, PyObject *args, PyObject *kwds) {
     self->task_spec = new ray::raylet::TaskSpecification(
         driver_id, parent_task_id, parent_counter, actor_creation_id,
         actor_creation_dummy_object_id, actor_id, actor_handle_id,
-        actor_counter, function_id, args, num_returns, required_resources);
+        actor_counter, function_id, args, num_returns, required_resources,
+        Language::PYTHON);
   }
 
   /* Set the task's execution dependencies. */

src/ray/raylet/lineage_cache_test.cc

@@ -114,7 +114,7 @@ static inline Task ExampleTask(const std::vector<ObjectID> &arguments,
   }
   auto spec = TaskSpecification(UniqueID::nil(), UniqueID::from_random(), 0,
                                 UniqueID::from_random(), task_arguments, num_returns,
-                                required_resources);
+                                required_resources, Language::PYTHON);
   auto execution_spec = TaskExecutionSpecification(std::vector<ObjectID>());
   execution_spec.IncrementNumForwards();
   Task task = Task(execution_spec, spec);

src/ray/raylet/task_dependency_manager_test.cc

@@ -76,7 +76,7 @@ static inline Task ExampleTask(const std::vector<ObjectID> &arguments,
   }
   auto spec = TaskSpecification(UniqueID::nil(), UniqueID::from_random(), 0,
                                 UniqueID::from_random(), task_arguments, num_returns,
-                                required_resources);
+                                required_resources, Language::PYTHON);
   auto execution_spec = TaskExecutionSpecification(std::vector<ObjectID>());
   execution_spec.IncrementNumForwards();
   Task task = Task(execution_spec, spec);

src/ray/raylet/task_spec.cc

@@ -45,10 +45,12 @@ TaskSpecification::TaskSpecification(
     const UniqueID &driver_id, const TaskID &parent_task_id, int64_t parent_counter,
     const FunctionID &function_id,
     const std::vector<std::shared_ptr<TaskArgument>> &task_arguments, int64_t num_returns,
-    const std::unordered_map<std::string, double> &required_resources)
+    const std::unordered_map<std::string, double> &required_resources,
+    const Language &language)
     : TaskSpecification(driver_id, parent_task_id, parent_counter, ActorID::nil(),
                         ObjectID::nil(), ActorID::nil(), ActorHandleID::nil(), -1,
-                        function_id, task_arguments, num_returns, required_resources) {}
+                        function_id, task_arguments, num_returns, required_resources,
+                        language) {}
 
 TaskSpecification::TaskSpecification(
     const UniqueID &driver_id, const TaskID &parent_task_id, int64_t parent_counter,
@@ -56,7 +58,8 @@ TaskSpecification::TaskSpecification(
     const ActorID &actor_id, const ActorHandleID &actor_handle_id, int64_t actor_counter,
     const FunctionID &function_id,
     const std::vector<std::shared_ptr<TaskArgument>> &task_arguments, int64_t num_returns,
-    const std::unordered_map<std::string, double> &required_resources)
+    const std::unordered_map<std::string, double> &required_resources,
+    const Language &language)
     : spec_() {
   flatbuffers::FlatBufferBuilder fbb;
 
@@ -87,6 +90,20 @@ TaskSpecification::TaskSpecification(
     returns.push_back(to_flatbuf(fbb, return_id));
   }
 
+  // convert Language to TaskLanguage
+  // TODO(raulchen): remove this once we get rid of legacy ray.
+  TaskLanguage task_language = TaskLanguage::PYTHON;
+  switch (language) {
+  case Language::PYTHON:
+    task_language = TaskLanguage::PYTHON;
+    break;
+  case Language::JAVA:
+    task_language = TaskLanguage::JAVA;
+    break;
+  default:
+    RAY_LOG(FATAL) << "Unknown language: " << static_cast<int32_t>(language);
+  }
+
   // Serialize the TaskSpecification.
   auto spec = CreateTaskInfo(
       fbb, to_flatbuf(fbb, driver_id), to_flatbuf(fbb, task_id),
@@ -94,7 +111,7 @@ TaskSpecification::TaskSpecification(
       to_flatbuf(fbb, actor_creation_dummy_object_id), to_flatbuf(fbb, actor_id),
       to_flatbuf(fbb, actor_handle_id), actor_counter, false,
       to_flatbuf(fbb, function_id), fbb.CreateVector(arguments),
-      fbb.CreateVector(returns), map_to_flatbuf(fbb, required_resources));
+      fbb.CreateVector(returns), map_to_flatbuf(fbb, required_resources), task_language);
   fbb.Finish(spec);
   AssignSpecification(fbb.GetBufferPointer(), fbb.GetSize());
 }
@@ -179,6 +196,22 @@ bool TaskSpecification::IsDriverTask() const {
   return FunctionId().is_nil();
 }
 
+Language TaskSpecification::GetLanguage() const {
+  auto message = flatbuffers::GetRoot<TaskInfo>(spec_.data());
+  // TODO(raulchen): remove this once we get rid of legacy ray.
+  auto language = message->language();
+  switch (language) {
+  case TaskLanguage::PYTHON:
+    return Language::PYTHON;
+  case TaskLanguage::JAVA:
+    return Language::JAVA;
+  default:
+    // This shouldn't be reachable.
+    RAY_LOG(FATAL) << "Unknown task language: " << static_cast<int32_t>(language);
+    return Language::PYTHON;
+  }
+}
+
 bool TaskSpecification::IsActorCreationTask() const {
   return !ActorCreationId().is_nil();
 }

src/ray/raylet/task_spec.h

@@ -98,7 +98,8 @@ class TaskSpecification {
                     int64_t parent_counter, const FunctionID &function_id,
                     const std::vector<std::shared_ptr<TaskArgument>> &arguments,
                     int64_t num_returns,
-                    const std::unordered_map<std::string, double> &required_resources);
+                    const std::unordered_map<std::string, double> &required_resources,
+                    const Language &language);
 
   TaskSpecification(const UniqueID &driver_id, const TaskID &parent_task_id,
                     int64_t parent_counter, const ActorID &actor_creation_id,
@@ -107,7 +108,8 @@ class TaskSpecification {
                     int64_t actor_counter, const FunctionID &function_id,
                     const std::vector<std::shared_ptr<TaskArgument>> &task_arguments,
                     int64_t num_returns,
-                    const std::unordered_map<std::string, double> &required_resources);
+                    const std::unordered_map<std::string, double> &required_resources,
+                    const Language &language);
 
   /// Deserialize a task specification from a flatbuffer's string data.
   ///
@@ -141,6 +143,7 @@ class TaskSpecification {
   double GetRequiredResource(const std::string &resource_name) const;
   const ResourceSet GetRequiredResources() const;
   bool IsDriverTask() const;
+  Language GetLanguage() const;
 
   // Methods specific to actor tasks.
   bool IsActorCreationTask() const;