[java] Remove multi-return API (#2724)

This commit is contained in:
Hao Chen
2018-08-26 15:04:54 +08:00
committed by Robert Nishihara
parent dbba7f2a53
commit 4f4bea086a
98 changed files with 615 additions and 7637 deletions
@@ -0,0 +1,19 @@
package org.ray.util.generator;
public abstract class BaseGenerator {
protected static final int MAX_PARAMETERS = 6;
protected StringBuilder sb;
protected void newLine(String line) {
sb.append(line).append("\n");
}
protected void indents(int numIndents) {
for (int i = 0; i < numIndents; i++) {
sb.append(" ");
}
}
}
@@ -1,38 +0,0 @@
package org.ray.util.generator;
import java.util.ArrayList;
import java.util.List;
/**
* Calculate all compositions for Parameter's count + Return's count, this class is used by code
* generators.
*/
public class Composition {
public static List<Tr> calculate(int maxT, int maxR) {
List<Tr> ret = new ArrayList<>();
for (int t = 0; t <= maxT; t++) {
// <= 0 for dynamic return count
// 0 for call_n returns RayMap<RID, x>
// -1 for call_n returns RayObject<>[N]
for (int r = -1; r <= maxR; r++) {
ret.add(new Tr(t, r));
}
}
return ret;
}
public static class Tr {
public final int tcount;
public final int rcount;
public Tr(int tcount, int rcount) {
super();
this.tcount = tcount;
this.rcount = rcount;
}
}
}
@@ -2,87 +2,47 @@ package org.ray.util.generator;
import java.io.IOException;
import org.ray.util.FileUtil;
import org.ray.util.generator.Composition.Tr;
/**
* Generate all classes in org.ray.api.funcs
* A util class that generates all the classes under org.ray.api.funcs package.
*/
public class FuncsGenerator {
public class FuncsGenerator extends BaseGenerator {
public static void main(String[] args) throws IOException {
String rootdir = System.getProperty("user.dir") + "/../api/src/main/java";
rootdir += "/org/ray/api/funcs";
generate(rootdir);
}
private String generate(int numParameters) {
sb = new StringBuilder();
private static void generate(String rootdir) throws IOException {
for (Tr tr : Composition.calculate(Share.MAX_T, Share.MAX_R)) {
String str = build(tr.tcount, tr.rcount);
String file = rootdir + "/RayFunc_" + tr.tcount + "_"
+ (tr.rcount <= 0 ? (tr.rcount == 0 ? "n" : "n_list") : tr.rcount) + ".java";
FileUtil.overrideFile(file, str);
System.err.println("override " + file);
String genericTypes = "";
String paramList = "";
for (int i = 0; i < numParameters; i++) {
genericTypes += "T" + i + ", ";
if (i > 0) {
paramList += ", ";
}
paramList += String.format("T%d t%d", i, i);
}
}
/*
* package org.ray.api.funcs;
*
* @FunctionalInterface
* public interface RayFunc_4_1<T0, T1, T2, T3, R0> extends RayFunc { R0
* apply();
*
* public static <R0> R0 execute(Object[] args) throws Throwable { String name =
* (String)args[args.length - 2]; assert (name.equals(RayFunc_0_1.class.getName())); byte[]
* funcBytes = (byte[])args[args.length - 1]; RayFunc_0_1<R0> f = (RayFunc_0_1<R0>)
* SerializationUtils.deserialize(funcBytes);
* return f.apply(); } }
*/
private static String build(int tcount, int rcount) {
StringBuilder sb = new StringBuilder();
String tname =
"Ray" + "Func_" + tcount + "_" + (rcount <= 0 ? (rcount == 0 ? "n" : "n_list") : rcount);
final String gname = tname + "<" + Share.buildClassDeclare(tcount, rcount) + ">";
newLine("// generated automatically, do not modify.");
newLine("");
newLine("package org.ray.api.funcs;");
newLine("");
newLine("@FunctionalInterface");
newLine(String.format("public interface RayFunc%d<%sR> extends RayFunc {",
numParameters, genericTypes));
indents(1);
newLine(String.format("R apply(%s);", paramList));
newLine("}");
sb.append("package org.ray.api.funcs;").append("\n");
if (rcount > 1) {
sb.append("import org.ray.api.returns.*;").append("\n");
}
if (rcount <= 0) {
sb.append("import java.util.Collection;").append("\n");
sb.append("import java.util.List;").append("\n");
sb.append("import java.util.Map;").append("\n");
}
sb.append("import org.ray.api.*;").append("\n");
sb.append("import org.ray.api.internal.*;").append("\n");
sb.append("import org.apache.commons.lang3.SerializationUtils;").append("\n");
sb.append("\n");
sb.append("@FunctionalInterface").append("\n");
sb.append("public interface ").append(gname).append(" extends RayFunc {")
.append("\n");
sb.append("\t").append(Share.buildFuncReturn(rcount)).append(" apply(")
.append(rcount == 0 ? ("Collection<RID> returnids" + (tcount > 0 ? ", " : "")) : "")
.append(Share.buildParameter(tcount, "T", null)).append(") throws Throwable;")
.append("\n");
sb.append("\t\n");
sb.append("\tpublic static " + "<").append(Share.buildClassDeclare(tcount, rcount))
.append(">").append(" ").append(Share.buildFuncReturn(rcount))
.append(" execute(Object[] args) throws Throwable {").append("\n");
sb.append("\t\tString name = (String)args[args.length - 2];").append("\n");
sb.append("\t\tassert (name.equals(").append(tname).append(".class.getName()));").append("\n");
sb.append("\t\tbyte[] funcBytes = (byte[])args[args.length - 1];").append("\n");
sb.append("\t\t").append(gname).append(" f = SerializationUtils.deserialize(funcBytes);")
.append("\n");
sb.append("\t\treturn f.apply(")
.append(rcount == 0 ? ("(Collection<RID>)args[0]" + (tcount > 0 ? ", " : "")) : "")
.append(Share.buildParameterUse2(tcount, rcount == 0 ? 1 : 0, "T", "args[", "]"))
.append(");").append("\n");
sb.append("\t}").append("\n");
sb.append("\t\n");
sb.append("}").append("\n");
return sb.toString();
}
public static void main(String[] args) throws IOException {
String root = System.getProperty("user.dir")
+ "/api/src/main/java/org/ray/api/funcs/";
FuncsGenerator generator = new FuncsGenerator();
for (int i = 0; i <= MAX_PARAMETERS; i++) {
String content = generator.generate(i);
FileUtil.overrideFile(root + "RayFunc" + i + ".java", content);
}
}
}
@@ -1,58 +0,0 @@
package org.ray.util.generator;
import java.io.IOException;
import org.ray.util.FileUtil;
/**
* Generate all classes in org.ray.api.returns.MultipleReturnsX
*/
public class MultipleReturnGenerator {
public static void main(String[] args) throws IOException {
String rootdir = System.getProperty("user.dir") + "/../api/src/main/java";
rootdir += "/org/ray/api/returns";
for (int r = 2; r <= Share.MAX_R; r++) {
String str = build(r);
String file = rootdir + "/MultipleReturns" + r + ".java";
FileUtil.overrideFile(file, str);
System.err.println("override " + file);
}
}
/**
* package org.ray.api.returns.
*/
private static String build(int rcount) {
StringBuilder sb = new StringBuilder();
sb.append("package org.ray.api.returns;").append("\n");
sb.append("import org.ray.api.*;").append("\n");
sb.append("@SuppressWarnings(\"unchecked\")");
sb.append("public class MultipleReturns").append(rcount).append("<")
.append(Share.buildClassDeclare(0, rcount)).append("> extends MultipleReturns {")
.append("\n");
sb.append("\tpublic MultipleReturns").append(rcount).append("(")
.append(Share.buildParameter(rcount, "R", null)).append(") {")
.append("\n");
sb.append("\t\tsuper(new Object[] { ").append(Share.buildParameterUse(rcount, "R"))
.append(" });")
.append("\n");
sb.append("\t}").append("\n");
for (int k = 0; k < rcount; k++) {
sb.append(buildGetter(k));
}
sb.append("}").append("\n");
return sb.toString();
}
/*
* @SuppressWarnings("unchecked") public R1 get1() { return (R1) this.values[1]; }
*/
private static String buildGetter(int index) {
return ("\tpublic R" + index + " get" + index + "() {\n")
+ "\t\treturn (R" + index + ") this.values[" + index + "];\n"
+ "\t}\n";
}
}
@@ -1,68 +0,0 @@
package org.ray.util.generator;
import java.io.IOException;
import org.ray.util.FileUtil;
/**
* Generate all classes in org.ray.api.returns.RayObjectsX
*/
public class RayObjectsGenerator {
public static void main(String[] args) throws IOException {
String rootdir = System.getProperty("user.dir") + "/../api/src/main/java";
rootdir += "/org/ray/api/returns";
for (int r = 2; r <= Share.MAX_R; r++) {
String str = build(r);
String file = rootdir + "/RayObjects" + r + ".java";
FileUtil.overrideFile(file, str);
System.err.println("override " + file);
}
}
/*
* package org.ray.api.returns;
*
* import org.ray.api.RayObject; import org.ray.spi.model.UniqueID;
*
* @SuppressWarnings({"rawtypes", "unchecked"}) public class RayObjects2<R0, R1> extends
* RayObjects {
*
* public RayObjects2(UniqueID[] ids) { super(ids); }
*
* public RayObjects2(RayObject objs[]) { super(objs); }
*
* public RayObject<R0> r0() { return objs[0]; }
*
* public RayObject<R1> r1() { return objs[1]; } }
*/
private static String build(int rcount) {
StringBuilder sb = new StringBuilder();
sb.append("package org.ray.api.returns;\n");
sb.append("import org.ray.api.*;\n");
sb.append("import org.ray.spi.model.UniqueID;\n");
sb.append("@SuppressWarnings({\"rawtypes\", \"unchecked\"})");
sb.append("public class RayObjects").append(rcount).append("<")
.append(Share.buildClassDeclare(0, rcount)).append("> extends RayObjects {")
.append("\n");
sb.append("\tpublic RayObjects").append(rcount).append("(UniqueID[] ids) {").append("\n");
sb.append("\t\tsuper(ids);").append("\n");
sb.append("\t}").append("\n");
sb.append("\tpublic RayObjects").append(rcount).append("(RayObject objs[]) {").append("\n");
sb.append("\t\tsuper(objs);").append("\n");
sb.append("\t}").append("\n");
for (int k = 0; k < rcount; k++) {
sb.append(buildGetter(k));
}
sb.append("}").append("\n");
return sb.toString();
}
private static String buildGetter(int index) {
return "\tpublic RayObject<R" + index + "> r" + index + "() {\n"
+ "\t\treturn objs[" + index + "];\n"
+ "\t}\n";
}
}
@@ -1,160 +1,77 @@
package org.ray.util.generator;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import java.util.ArrayList;
import java.util.List;
import org.ray.util.FileUtil;
import org.ray.util.generator.Composition.Tr;
/**
* Generate Rpc.java
* A util class that generates Rpc.java
*/
public class RpcGenerator {
public class RpcGenerator extends BaseGenerator {
private String build() {
sb = new StringBuilder();
newLine("// generated automatically, do not modify.");
newLine("");
newLine("package org.ray.api;");
newLine("");
newLine("import org.ray.api.funcs.*;");
newLine("");
newLine("@SuppressWarnings({\"rawtypes\", \"unchecked\"})");
newLine("class Rpc {");
for (int i = 0; i <= 6; i++) {
buildCalls(i);
}
newLine("}");
return sb.toString();
}
private void buildCalls(int numParameters) {
String funcClass = "RayFunc" + numParameters;
String genericTypes = "";
String callList = "";
for (int i = 1; i <= numParameters; i++) {
genericTypes += "T" + i + ", ";
callList += ", t" + i;
}
String body = String.format("return Ray.internal().call(f%s);", callList);
for (String param : generateParameters(numParameters)) {
indents(1);
newLine(String.format(
"public static <%sR> RayObject<R> call(%s<%sR> f%s) {",
genericTypes, funcClass, genericTypes, param
));
indents(2);
newLine(body);
indents(1);
newLine("}");
}
}
private List<String> generateParameters(int numParameters) {
List<String> res = new ArrayList<>();
dfs(1, numParameters, "", res);
return res;
}
private void dfs(int pos, int max, String cur, List<String> res) {
if (pos > max) {
res.add(cur);
return;
}
cur += ", ";
String nextParameter = String.format("T%d t%d", pos, pos);
dfs(pos + 1, max, cur + nextParameter, res);
nextParameter = String.format("RayObject<T%d> t%d", pos, pos);
dfs(pos + 1, max, cur + nextParameter, res);
}
public static void main(String[] args) throws IOException {
String rootdir = System.getProperty("user.dir") + "/../api/src/main/java";
rootdir += "/org/ray/api/Rpc.java";
FileUtil.overrideFile(rootdir, build());
String path = System.getProperty("user.dir") + "/api/src/main/java";
path += "/org/ray/api/Rpc.java";
FileUtil.overrideFile(path, new RpcGenerator().build());
}
private static String build() {
StringBuilder sb = new StringBuilder();
sb.append("package org.ray.api.internal;\n");
sb.append("import org.ray.api.funcs.*;\n");
sb.append("import org.ray.api.returns.*;\n");
sb.append("import org.ray.api.*;\n");
sb.append("import java.util.Collection;\n");
sb.append("import java.util.Map;\n");
sb.append("@SuppressWarnings({\"rawtypes\", \"unchecked\"})\n");
sb.append("class Rpc {\n");
for (Tr tr : Composition.calculate(Share.MAX_T, Share.MAX_R)) {
buildCall(sb, tr.tcount, tr.rcount);
}
sb.append("}\n");
return sb.toString();
}
private static void buildCall(StringBuilder sb, int tcount, int rcount) {
for (Set<Integer> whichTisFuture : whichTisFutureComposition(tcount)) {
sb.append(buildCall(tcount, rcount, whichTisFuture));
}
}
private static String buildCall(int tcount, int rcount, Set<Integer> whichTisFuture) {
StringBuilder sb = new StringBuilder();
String parameter = (tcount == 0 ? ""
: ", " + Share.buildParameter(tcount, "T", whichTisFuture));
sb.append("\tpublic static <").append(Share.buildClassDeclare(tcount, rcount)).append("> ")
.append(Share.buildRpcReturn(rcount)).append(" call")
.append(rcount == 1 ? "" : (rcount <= 0 ? "_n" : ("_" + rcount))).append("(RayFunc_")
.append(tcount).append("_")
.append(rcount <= 0 ? (rcount == 0 ? "n" : "n_list") : rcount).append("<")
.append(Share.buildClassDeclare(tcount, rcount)).append("> f").append(
rcount <= 0 ? (rcount == 0 ? ", Collection<RID> returnids" : ", Integer returnCount")
: "").append(parameter).append(") {\n");
String nulls = Share.buildRepeat("null",
tcount + (rcount == 0 ? 1/*for first arg map*/ : 0));
String parameterUse = (tcount == 0 ? "" : (", " + Share.buildParameterUse(tcount, "T")));
String labmdaUse = "RayFunc_"
+ tcount + "_" + (rcount <= 0 ? (rcount == 0 ? "n" : "n_list") : rcount)
+ ".class, f";
sb.append("\t\tif (Ray.Parameters().remoteLambda()) {\n");
if (rcount == 1) {
sb.append("\t\t\treturn Ray.internal().call(null, ").append(labmdaUse).append(", 1")
.append(parameterUse).append(").objs[0];")
.append("\n");
} else if (rcount == 0) {
sb.append("\t\t\treturn Ray.internal().callWithReturnLabels(null, ")
.append(labmdaUse).append(", returnids").append(parameterUse).append(");")
.append("\n");
} else if (rcount < 0) {
sb.append("\t\t\treturn Ray.internal().callWithReturnIndices(null, ")
.append(labmdaUse).append(", returnCount").append(parameterUse).append(");")
.append("\n");
} else {
sb.append("\t\t\treturn new RayObjects").append(rcount)
.append("(Ray.internal().call(null, ").append(labmdaUse).append(", ").append(rcount)
.append(parameterUse).append(").objs);")
.append("\n");
}
sb.append("\t\t} else {\n");
if (rcount == 1) {
sb.append("\t\t\treturn Ray.internal().call(null, () -> f.apply(").append(nulls)
.append("), 1").append(parameterUse).append(").objs[0];")
.append("\n");
} else if (rcount == 0) {
sb.append("\t\t\treturn Ray.internal().callWithReturnLabels(null, () -> f.apply(")
.append(nulls).append("), returnids").append(parameterUse).append(");")
.append("\n");
} else if (rcount < 0) {
sb.append("\t\t\treturn Ray.internal().callWithReturnIndices(null, () -> f.apply(")
.append(nulls).append("), returnCount").append(parameterUse).append(");")
.append("\n");
} else {
sb.append("\t\t\treturn new RayObjects").append(rcount)
.append("(Ray.internal().call(null, () -> f.apply(").append(nulls).append("), ")
.append(rcount).append(parameterUse).append(").objs);")
.append("\n");
}
sb.append("\t\t}\n");
sb.append("\t}\n");
return sb.toString();
}
private static Set<Set<Integer>> whichTisFutureComposition(int tcount) {
Set<Set<Integer>> ret = new HashSet<>();
Set<Integer> n = new HashSet<>();
for (int k = 0; k < tcount; k++) {
n.add(k);
}
for (int k = 0; k <= tcount; k++) {
ret.addAll(cnn(n, k));
}
return ret;
}
//pick n numbers in N
private static Set<Set<Integer>> cnn(Set<Integer> bigN, int n) {
C c = new C();
for (int k = 0; k < n; k++) {
c.mul(bigN);
}
return c.vc;
}
static class C {
Set<Set<Integer>> vc;
public C() {
vc = new HashSet<>();
vc.add(new HashSet<>());
}
void mul(Set<Integer> ns) {
Set<Set<Integer>> ret = new HashSet<>();
for (int n : ns) {
ret.addAll(mul(n));
}
this.vc = ret;
}
Set<Set<Integer>> mul(int n) {
Set<Set<Integer>> ret = new HashSet<>();
for (Set<Integer> s : vc) {
if (s.contains(n)) {
continue;
}
Set<Integer> ns = new HashSet<>(s);
ns.add(n);
ret.add(ns);
}
return ret;
}
}
}
@@ -1,134 +0,0 @@
package org.ray.util.generator;
import java.util.Set;
/**
* Share util for generators.
*/
public class Share {
public static final int MAX_T = 6;
public static final int MAX_R = 4;
/**
* T0, T1, T2, T3, R.
*/
public static String buildClassDeclare(int tcount, int rcount) {
StringBuilder sb = new StringBuilder();
for (int k = 0; k < tcount; k++) {
sb.append("T").append(k).append(", ");
}
if (rcount == 0) {
sb.append("R, RID");
} else if (rcount < 0) {
assert (rcount == -1);
sb.append("R");
} else {
for (int k = 0; k < rcount; k++) {
sb.append("R").append(k).append(", ");
}
sb.deleteCharAt(sb.length() - 1);
sb.deleteCharAt(sb.length() - 1);
}
return sb.toString();
}
/**
* T0 t0, T1 t1, T2 t2, T3 t3.
*/
public static String buildParameter(int tcount, String tr, Set<Integer> whichTisFuture) {
StringBuilder sb = new StringBuilder();
for (int k = 0; k < tcount; k++) {
sb.append(whichTisFuture != null && whichTisFuture.contains(k)
? "RayObject<" + (tr + k) + ">" : (tr + k)).append(" ").append(tr.toLowerCase())
.append(k).append(", ");
}
if (tcount > 0) {
sb.deleteCharAt(sb.length() - 1);
sb.deleteCharAt(sb.length() - 1);
}
return sb.toString();
}
/**
* t0, t1, t2.
*/
public static String buildParameterUse(int tcount, String tr) {
StringBuilder sb = new StringBuilder();
for (int k = 0; k < tcount; k++) {
sb.append(tr.toLowerCase()).append(k).append(", ");
}
if (tcount > 0) {
sb.deleteCharAt(sb.length() - 1);
sb.deleteCharAt(sb.length() - 1);
}
return sb.toString();
}
public static String buildParameterUse2(int tcount, int startIndex, String tr, String pre,
String post) {
StringBuilder sb = new StringBuilder();
for (int k = 0; k < tcount; k++) {
sb.append("(").append(tr).append(k).append(")").append(pre).append(k + startIndex)
.append(post).append(", ");
}
if (tcount > 0) {
sb.deleteCharAt(sb.length() - 1);
sb.deleteCharAt(sb.length() - 1);
}
return sb.toString();
}
public static String buildFuncReturn(int rcount) {
if (rcount == 0) {
return "Map<RID, R>";
} else if (rcount < 0) {
assert (-1 == rcount);
return "List<R>";
}
if (rcount == 1) {
return "R0";
}
StringBuilder sb = new StringBuilder();
for (int k = 0; k < rcount; k++) {
sb.append("R").append(k).append(", ");
}
sb.deleteCharAt(sb.length() - 1);
sb.deleteCharAt(sb.length() - 1);
return "MultipleReturns" + rcount + "<" + sb.toString() + ">";
}
public static String buildRpcReturn(int rcount) {
if (rcount == 0) {
return "RayMap<RID, R>";
} else if (rcount < 0) {
assert (rcount == -1);
return "RayList<R>";
}
if (rcount == 1) {
return "RayObject<R0>";
}
StringBuilder sb = new StringBuilder();
for (int k = 0; k < rcount; k++) {
sb.append("R").append(k).append(", ");
}
sb.deleteCharAt(sb.length() - 1);
sb.deleteCharAt(sb.length() - 1);
return "RayObjects" + rcount + "<" + sb.toString() + ">";
}
public static String buildRepeat(String toRepeat, int count) {
StringBuilder ret = new StringBuilder();
for (int k = 0; k < count; k++) {
ret.append(toRepeat).append(", ");
}
if (count > 0) {
ret.deleteCharAt(ret.length() - 1);
ret.deleteCharAt(ret.length() - 1);
}
return ret.toString();
}
}
@@ -1,5 +1,5 @@
# define default properties here
logging.level=INFO
logging.level=WARN
logging.path=./run/logs
logging.file.name=core
logging.max.log.file.num=10