wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-10 20:19:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

[JavaWorker] Enable java worker support (#2094)

Browse Source 
* Enable java worker support
--------------------------
This commit includes a tailored version of the Java worker implementation from Ant Financial.
The changes for build system, python module, src module and arrow are in other commits, this commit consists of the following modules:
 - java/api: Ray API definition
 - java/common: utilities
 - java/hook: binary rewrite of the Java byte-code for remote execution
 - java/runtime-common: common implementation of the runtime in worker
 - java/runtime-dev: a pure-java mock implementation of the runtime for fast development
 - java/runtime-native: a native implementation of the runtime
 - java/test: various tests

Contributors for this work:
 Guyang Song, Peng Cao, Senlin Zhu,Xiaoying Chu, Yiming Yu, Yujie Liu, Zhenyu Guo

* change the format of java help document from markdown to RST

* update the vesion of Arrow for java worker

* adapt the new version of plasma java client from arrow which use byte[] instead of custom type

* add java worker test to ci

* add the example module for better usage guide
This commit is contained in:
Yujie Liu
2018-05-27 05:38:50 +08:00
committed by Philipp Moritz
parent 74cca3b284
commit a8d3c057c1
193 changed files with 22675 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/common/pom.xml
+40
View File
@@ -0,0 +1,40 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<parent>
<groupId>org.ray.parent</groupId>
<artifactId>ray-superpom</artifactId>
<version>1.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.ray</groupId>
<artifactId>ray-common</artifactId>
<name>java common and util for ray</name>
<description>java common and util for ray</description>
<url></url>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
<dependency>
<groupId>quartz</groupId>
<artifactId>quartz</artifactId>
<version>1.5.2</version>
</dependency>
<dependency>
<groupId>org.ini4j</groupId>
<artifactId>ini4j</artifactId>
<version>0.5.2</version>
</dependency>
</dependencies>
</project>
java/common/src/main/java/org/ray/util/CommonUtil.java
+19
View File
@@ -0,0 +1,19 @@
package org.ray.util;
import java.util.Random;
/**
* Common utilities
*/
public class CommonUtil {
private static final Random seed = new Random();
/**
* Get random number between 0 and (max-1)
*/
public static int getRandom(int max) {
return Math.abs(seed.nextInt() % max);
}
}
java/common/src/main/java/org/ray/util/FileUtil.java
+134
View File
@@ -0,0 +1,134 @@
package org.ray.util;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Scanner;
public class FileUtil {
public static String getFilename(String logPath) {
if (logPath != null && !logPath.isEmpty()) {
int lastPos = logPath.lastIndexOf('/');
if (lastPos != -1) {
return logPath.substring(lastPos + 1);
} else {
return logPath;
}
}
return null;
}
public static boolean deleteFile(String filePath) {
File file = new File(filePath);
if (!file.exists()) {
return true;
} else {
if (file.isFile()) {
return file.delete();
} else {
for (File f : file.listFiles()) {
deleteFile(f.getAbsolutePath());
}
return file.delete();
}
}
}
public static void mkDir(File dir) {
if (dir.exists()) {
return;
}
if (dir.getParentFile().exists()) {
dir.mkdir();
} else {
mkDir(dir.getParentFile());
dir.mkdir();
}
}
public static void mkDirAndFile(File file) throws IOException {
if (file.exists()) {
return;
}
if (!file.getParentFile().exists()) {
mkDir(file.getParentFile());
}
file.createNewFile();
}
public static String readResourceFile(String fileName) throws FileNotFoundException {
ClassLoader classLoader = FileUtil.class.getClassLoader();
File file = new File(classLoader.getResource(fileName).getFile());
StringBuilder result = new StringBuilder();
try (Scanner scanner = new Scanner(file)) {
//Get file from resources folder
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
result.append(line).append("\n");
}
return result.toString();
}
}
public static void overrideFile(String file, String str) throws IOException {
try (FileWriter fw = new FileWriter(file)) {
fw.write(str);
}
}
public static boolean createDir(String dirName, boolean failIfExist) {
File wdir = new File(dirName);
if (wdir.isFile()) {
return false;
}
if (!wdir.exists()) {
wdir.mkdirs();
} else if (failIfExist) {
return false;
}
return true;
}
public static void bytesToFile(byte[] bytes, String name) throws IOException {
Path path = Paths.get(name);
Files.write(path, bytes);
}
public static byte[] fileToBytes(String name) throws IOException {
Path path = Paths.get(name);
return Files.readAllBytes(path);
}
/**
* If the given string is the empty string, then the result is the current directory.
*
* @param rawDir a path in any legal form, such as a relative path
* @return the absolute and unique path in String
*/
public static String getCanonicalDirectory(final String rawDir) throws IOException {
String dir = rawDir.length() == 0 ? "." : rawDir;
// create working dir if necessary
File dd = new File(dir);
if (!dd.exists()) {
dd.mkdirs();
}
if (!dir.startsWith("/")) {
dir = dd.getCanonicalPath();
}
return dir;
}
}
java/common/src/main/java/org/ray/util/MD5Digestor.java
+30
View File
@@ -0,0 +1,30 @@
package org.ray.util;
import java.nio.ByteBuffer;
import java.security.MessageDigest;
import org.ray.util.logger.RayLog;
public class MD5Digestor {
private static final ThreadLocal<MessageDigest> md = ThreadLocal.withInitial(() -> {
try {
return MessageDigest.getInstance("MD5");
} catch (Exception e) {
RayLog.core.error("cannot get MD5 MessageDigest", e);
throw new RuntimeException("cannot get MD5 digest", e);
}
});
private static final ThreadLocal<ByteBuffer> longBuffer = ThreadLocal
.withInitial(() -> ByteBuffer.allocate(Long.SIZE / Byte.SIZE));
public static byte[] digest(byte[] src, long addIndex) {
MessageDigest dg = md.get();
longBuffer.get().clear();
dg.reset();
dg.update(src);
dg.update(longBuffer.get().putLong(addIndex).array());
return dg.digest();
}
}
java/common/src/main/java/org/ray/util/NetworkUtil.java
+62
View File
@@ -0,0 +1,62 @@
package org.ray.util;
import java.io.IOException;
import java.net.DatagramSocket;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.NetworkInterface;
import java.net.ServerSocket;
import java.util.Enumeration;
import org.ray.util.logger.RayLog;
public class NetworkUtil {
public static String getIpAddress(String interfaceName) {
try {
Enumeration<NetworkInterface> interfaces = NetworkInterface.getNetworkInterfaces();
while (interfaces.hasMoreElements()) {
NetworkInterface current = interfaces.nextElement();
if (!current.isUp() || current.isLoopback() || current.isVirtual()) {
continue;
}
if (!StringUtil.isNullOrEmpty(interfaceName) && !interfaceName
.equals(current.getDisplayName())) {
continue;
}
Enumeration<InetAddress> addresses = current.getInetAddresses();
while (addresses.hasMoreElements()) {
InetAddress addr = addresses.nextElement();
if (addr.isLoopbackAddress()) {
continue;
}
if (addr instanceof Inet6Address) {
continue;
}
return addr.getHostAddress();
}
}
RayLog.core.warn("you may need to correctly specify [ray.java] net_interface in config");
} catch (Exception e) {
RayLog.core.error("Can't get our ip address, use 127.0.0.1 as default.", e);
}
return "127.0.0.1";
}
public static boolean isPortAvailable(int port) {
if (port < 1 || port > 65535) {
throw new IllegalArgumentException("Invalid start port: " + port);
}
try (ServerSocket ss = new ServerSocket(port); DatagramSocket ds = new DatagramSocket(port)) {
ss.setReuseAddress(true);
ds.setReuseAddress(true);
return true;
} catch (IOException ignored) {
}
/* should not be thrown */
return false;
}
}
java/common/src/main/java/org/ray/util/ObjectUtil.java
+24
View File
@@ -0,0 +1,24 @@
package org.ray.util;
import java.lang.reflect.InvocationTargetException;
public class ObjectUtil {
public static <T> T newObject(Class<T> cls) {
try {
return cls.getConstructor().newInstance();
} catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
e.printStackTrace();
return null;
}
}
public static boolean[] toBooleanArray(Object[] vs) {
boolean[] nvs = new boolean[vs.length];
for (int i = 0; i < vs.length; i++) {
nvs[i] = (boolean) vs[i];
}
return nvs;
}
}
java/common/src/main/java/org/ray/util/RemoteBiFunction.java
+10
View File
@@ -0,0 +1,10 @@
package org.ray.util;
import java.io.Serializable;
import java.util.function.BiFunction;
@FunctionalInterface
public interface RemoteBiFunction<T1, T2, O> extends BiFunction<T1, T2, O>, Serializable {
}
java/common/src/main/java/org/ray/util/RemoteFunction.java
+9
View File
@@ -0,0 +1,9 @@
package org.ray.util;
import java.io.Serializable;
import java.util.function.Function;
@FunctionalInterface
public interface RemoteFunction<I, O> extends Function<I, O>, Serializable {
}
java/common/src/main/java/org/ray/util/Sha1Digestor.java
+30
View File
@@ -0,0 +1,30 @@
package org.ray.util;
import java.nio.ByteBuffer;
import java.security.MessageDigest;
import org.ray.util.logger.RayLog;
public class Sha1Digestor {
private static final ThreadLocal<MessageDigest> md = ThreadLocal.withInitial(() -> {
try {
return MessageDigest.getInstance("SHA1");
} catch (Exception e) {
RayLog.core.error("cannot get SHA1 MessageDigest", e);
throw new RuntimeException("cannot get SHA1 digest", e);
}
});
private static final ThreadLocal<ByteBuffer> longBuffer = ThreadLocal
.withInitial(() -> ByteBuffer.allocate(Long.SIZE / Byte.SIZE));
public static byte[] digest(byte[] src, long addIndex) {
MessageDigest dg = md.get();
longBuffer.get().clear();
dg.reset();
dg.update(src);
dg.update(longBuffer.get().putLong(addIndex).array());
return dg.digest();
}
}
java/common/src/main/java/org/ray/util/StringUtil.java
+134
View File
@@ -0,0 +1,134 @@
package org.ray.util;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Vector;
public class StringUtil {
// Holds the start of an element and which brace started it.
private static class Start {
// The brace number from the braces string in use.
final int brace;
// The position in the string it was seen.
final int pos;
// Constructor.
public Start(int brace, int pos) {
this.brace = brace;
this.pos = pos;
}
}
/**
* @param s input string
* @param splitters common splitters
* @param open open braces
* @param close close braces
* @return output array list
*/
public static Vector<String> Split(String s, String splitters, String open, String close) {
// The splits.
Vector<String> split = new Vector<>();
// The stack.
ArrayList<Start> stack = new ArrayList<>();
int lastPos = 0;
// Walk the string.
for (int i = 0; i < s.length(); i++) {
// Get the char there.
char ch = s.charAt(i);
// Is it an open brace?
int o = open.indexOf(ch);
// Is it a close brace?
int c = close.indexOf(ch);
// Is it a splitter?
int sp = splitters.indexOf(ch);
if (stack.size() == 0 && sp >= 0) {
if (i == lastPos) {
++lastPos;
continue;
}
split.add(s.substring(lastPos, i));
lastPos = i + 1;
} else if (o >= 0 && (c < 0 || stack.size() == 0)) {
// Its an open! Push it.
stack.add(new Start(o, i));
} else if (c >= 0 && stack.size() > 0) {
// Pop (if matches).
int tosPos = stack.size() - 1;
Start tos = stack.get(tosPos);
// Does the brace match?
if (tos.brace == c) {
// Done with that one.
stack.remove(tosPos);
}
}
}
if (lastPos < s.length()) {
split.add(s.substring(lastPos, s.length()));
}
// build removal filter set
HashSet<Character> removals = new HashSet<>();
for (int i = 0; i < splitters.length(); i++) {
removals.add(splitters.charAt(i));
}
for (int i = 0; i < open.length(); i++) {
removals.add(open.charAt(i));
}
for (int i = 0; i < close.length(); i++) {
removals.add(close.charAt(i));
}
// apply removal filter set
for (int i = 0; i < split.size(); i++) {
String cs = split.get(i);
// remove heading chars
int j;
for (j = 0; j < cs.length(); j++) {
if (!removals.contains(cs.charAt(j))) {
break;
}
}
cs = cs.substring(j);
// remove tail chars
for (j = cs.length() - 1; j >= 0; j--) {
if (!removals.contains(cs.charAt(j))) {
break;
}
}
cs = cs.substring(0, j + 1);
// reset cs
split.set(i, cs);
}
return split;
}
public static boolean isNullOrEmpty(String s) {
return s == null || s.length() == 0;
}
public static <T> String mergeArray(T[] objs, String concatenator) {
StringBuilder sb = new StringBuilder();
for (T obj : objs) {
sb.append(obj).append(concatenator);
}
return objs.length == 0 ? "" : sb.substring(0, sb.length() - concatenator.length());
}
}
java/common/src/main/java/org/ray/util/SystemUtil.java
+64
View File
@@ -0,0 +1,64 @@
package org.ray.util;
import java.lang.management.ManagementFactory;
import java.lang.management.RuntimeMXBean;
import java.util.concurrent.locks.ReentrantLock;
import org.ray.util.logger.RayLog;
/**
* some utilities for system process
*/
public class SystemUtil {
public static String userHome() {
return System.getProperty("user.home");
}
public static String userDir() {
return System.getProperty("user.dir");
}
public static boolean startWithJar(Class<?> cls) {
return cls.getResource(cls.getSimpleName() + ".class").getFile().split("!")[0].endsWith(".jar");
}
public static boolean startWithJar(String clsName) {
Class<?> cls;
try {
cls = Class.forName(clsName);
return cls.getResource(cls.getSimpleName() + ".class").getFile().split("!")[0]
.endsWith(".jar");
} catch (ClassNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
RayLog.core.error("error at SystemUtil startWithJar", e);
return false;
}
}
static Integer pid;
static final ReentrantLock pidlock = new ReentrantLock();
public static int pid() {
if (pid == null) {
pidlock.lock();
try {
if (pid == null) {
RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();
String name = runtime.getName();
int index = name.indexOf("@");
if (index != -1) {
pid = Integer.parseInt(name.substring(0, index));
} else {
throw new RuntimeException("parse pid error:" + name);
}
}
} finally {
pidlock.unlock();
}
}
return pid;
}
}
java/common/src/main/java/org/ray/util/config/AConfig.java
+36
View File
@@ -0,0 +1,36 @@
package org.ray.util.config;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Annotate a field as a ray configuration item.
*/
@Target({ElementType.FIELD})
@Retention(RetentionPolicy.RUNTIME)
public @interface AConfig {
/**
* comments for this configuration field
*/
String comment();
/**
* when the config is an array list, a splitter set is specified, e.g., " \t" to use ' ' and '\t'
* as possible splits
*/
String splitters() default ", \t";
/**
* indirect with value as the new section name, the field name remains the same
*/
String defaultIndirectSectionName() default "";
/**
* see ConfigReader.getIndirectStringArray this config tells which is the default
* indirectSectionName in that function
*/
String defaultArrayIndirectSectionName() default "";
}
java/common/src/main/java/org/ray/util/config/ConfigItem.java
+15
View File
@@ -0,0 +1,15 @@
package org.ray.util.config;
/**
* A ray configuration item of type {@code T}.
*/
public class ConfigItem<T> {
public String key;
public String oriValue;
public T defaultValue;
public String desc;
}
java/common/src/main/java/org/ray/util/config/ConfigReader.java
+385
View File
@@ -0,0 +1,385 @@
package org.ray.util.config;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Vector;
import org.ini4j.Config;
import org.ini4j.Ini;
import org.ini4j.Profile;
import org.ray.util.ObjectUtil;
import org.ray.util.StringUtil;
/**
* Loads configurations from a file.
*/
public class ConfigReader {
private final CurrentUseConfig currentUseConfig = new CurrentUseConfig();
private final Ini ini = new Ini();
private String file = "";
public ConfigReader(String filePath) throws Exception {
this(filePath, null);
}
public ConfigReader(String filePath, String updateConfigStr) throws Exception {
System.out.println("Build ConfigReader, the file path " + filePath + " ,the update config str "
+ updateConfigStr);
try {
loadConfigFile(filePath);
updateConfigFile(updateConfigStr);
} catch (Exception e) {
e.printStackTrace();
throw e;
}
}
public String filePath() {
return file;
}
private void loadConfigFile(String filePath) throws Exception {
this.currentUseConfig.filePath = filePath;
String configFileDir = (new File(filePath)).getAbsoluteFile().getParent();
byte[] encoded = Files.readAllBytes(Paths.get(filePath));
String content = new String(encoded, StandardCharsets.UTF_8);
content = content.replaceAll("%CONFIG_FILE_DIR%", configFileDir);
InputStream fis = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
Config config = new Config();
ini.setConfig(config);
ini.load(fis);
file = currentUseConfig.filePath;
}
private void updateConfigFile(String updateConfigStr) {
if (updateConfigStr == null) {
return;
}
String[] updateConfigArray = updateConfigStr.split(";");
for (String currentUpdateConfig : updateConfigArray) {
if (StringUtil.isNullOrEmpty(currentUpdateConfig)) {
continue;
}
String[] currentUpdateConfigArray = currentUpdateConfig.split("=");
String sectionAndItemKey;
String value = "";
if (currentUpdateConfigArray.length == 2) {
sectionAndItemKey = currentUpdateConfigArray[0];
value = currentUpdateConfigArray[1];
} else if (currentUpdateConfigArray.length == 1) {
sectionAndItemKey = currentUpdateConfigArray[0];
} else {
String errorMsg = "invalid config (must be of k=v or k or k=): " + currentUpdateConfig;
System.err.println(errorMsg);
throw new RuntimeException(errorMsg);
}
int splitOffset = sectionAndItemKey.lastIndexOf(".");
int len = sectionAndItemKey.length();
if (splitOffset < 1 || splitOffset == len - 1) {
String errorMsg =
"invalid config (no '.' found for section name and key):" + currentUpdateConfig;
System.err.println(errorMsg);
throw new RuntimeException(errorMsg);
}
String sectionKey = sectionAndItemKey.substring(0, splitOffset);
String itemKey = sectionAndItemKey.substring(splitOffset + 1);
if (ini.containsKey(sectionKey)) {
ini.get(sectionKey).put(itemKey, value);
} else {
ini.add(sectionKey, itemKey, value);
}
}
}
public CurrentUseConfig getCurrentUseConfig() {
return currentUseConfig;
}
private synchronized <T> String getOriValue(String sectionKey, String configKey, T defaultValue,
String deptr) {
if (null == deptr) {
throw new RuntimeException("desc must not be empty of the key:" + configKey);
}
Profile.Section section = ini.get(sectionKey);
String oriValue = null;
if (section != null && section.containsKey(configKey)) {
oriValue = section.get(configKey);
}
if (!currentUseConfig.sectionMap.containsKey(sectionKey)) {
ConfigSection configSection = new ConfigSection();
configSection.sectionKey = sectionKey;
updateConfigSection(configSection, configKey, defaultValue, deptr, oriValue);
currentUseConfig.sectionMap.put(sectionKey, configSection);
} else if (!currentUseConfig.sectionMap.get(sectionKey).itemMap.containsKey(configKey)) {
ConfigSection configSection = currentUseConfig.sectionMap.get(sectionKey);
updateConfigSection(configSection, configKey, defaultValue, deptr, oriValue);
}
return oriValue;
}
private <T> void updateConfigSection(ConfigSection configSection, String configKey,
T defaultValue, String deptr, String oriValue) {
ConfigItem<T> configItem = new ConfigItem<>();
configItem.defaultValue = defaultValue;
configItem.key = configKey;
configItem.oriValue = oriValue;
configItem.desc = deptr;
configSection.itemMap.put(configKey, configItem);
}
public String getStringValue(String sectionKey, String configKey, String defaultValue,
String dsptr) {
String value = getOriValue(sectionKey, configKey, defaultValue, dsptr);
if (value != null) {
return value;
} else {
return defaultValue;
}
}
public boolean getBooleanValue(String sectionKey, String configKey, boolean defaultValue,
String dsptr) {
String value = getOriValue(sectionKey, configKey, defaultValue, dsptr);
if (value != null) {
if (value.length() == 0) {
return defaultValue;
} else {
return Boolean.valueOf(value);
}
} else {
return defaultValue;
}
}
public int getIntegerValue(String sectionKey, String configKey, int defaultValue, String dsptr) {
String value = getOriValue(sectionKey, configKey, defaultValue, dsptr);
if (value != null) {
if (value.length() == 0) {
return defaultValue;
} else {
return Integer.valueOf(value);
}
} else {
return defaultValue;
}
}
public long getLongValue(String sectionKey, String configKey, long defaultValue, String dsptr) {
String value = getOriValue(sectionKey, configKey, defaultValue, dsptr);
if (value != null) {
if (value.length() == 0) {
return defaultValue;
} else {
return Long.valueOf(value);
}
} else {
return defaultValue;
}
}
public double getDoubleValue(String sectionKey, String configKey, double defaultValue,
String dsptr) {
String value = getOriValue(sectionKey, configKey, defaultValue, dsptr);
if (value != null) {
if (value.length() == 0) {
return defaultValue;
} else {
return Double.valueOf(value);
}
} else {
return defaultValue;
}
}
public int[] getIntegerArray(String sectionKey, String configKey, int[] defaultValue,
String dsptr) {
String value = getOriValue(sectionKey, configKey, defaultValue, dsptr);
int[] array = defaultValue;
if (value != null) {
String[] list = value.split(",");
array = new int[list.length];
for (int i = 0; i < list.length; i++) {
array[i] = Integer.valueOf(list[i]);
}
}
return array;
}
/**
* get a string list from a whole section as keys e.g., [core] data_dirs = local.dirs # or
* cluster.dirs
*
* [local.dirs] /home/xxx/1 /home/yyy/2
*
* [cluster.dirs] ...
*
* @param sectionKey e.g., core
* @param configKey e.g., data_dirs
* @param indirectSectionName e.g., cluster.dirs
* @return string list
*/
public String[] getIndirectStringArray(String sectionKey, String configKey,
String indirectSectionName, String dsptr) {
String s = getStringValue(sectionKey, configKey, indirectSectionName, dsptr);
Profile.Section section = ini.get(s);
if (section == null) {
return new String[]{};
} else {
return section.keySet().toArray(new String[]{});
}
}
public <T> void readObject(String sectionKey, T obj, T defaultValues) {
for (Field fld : obj.getClass().getFields()) {
Object defaultFldValue;
try {
defaultFldValue = defaultValues != null ? fld.get(defaultValues) : null;
} catch (IllegalArgumentException | IllegalAccessException e) {
defaultFldValue = null;
}
String section = sectionKey;
String comment;
String splitters = ", \t";
String defaultArrayIndirectSectionName;
AConfig[] anns = fld.getAnnotationsByType(AConfig.class);
if (anns.length > 0) {
comment = anns[0].comment();
if (!StringUtil.isNullOrEmpty(anns[0].splitters())) {
splitters = anns[0].splitters();
}
defaultArrayIndirectSectionName = anns[0].defaultArrayIndirectSectionName();
// redirect the section if necessary
if (!StringUtil.isNullOrEmpty(anns[0].defaultIndirectSectionName())) {
section = this
.getStringValue(sectionKey, fld.getName(), anns[0].defaultIndirectSectionName(),
comment);
}
} else {
throw new RuntimeException("unspecified comment, please use @AConfig(comment = xxxx) for "
+ obj.getClass().getName() + "." + fld.getName() + "'s configuration descriptions ");
}
try {
if (fld.getType().isPrimitive()) {
if (fld.getType().equals(boolean.class)) {
boolean v = getBooleanValue(section, fld.getName(), (boolean) defaultFldValue, comment);
fld.set(obj, v);
} else if (fld.getType().equals(float.class)) {
float v = (float) getDoubleValue(section, fld.getName(),
(double) (float) defaultFldValue, comment);
fld.set(obj, v);
} else if (fld.getType().equals(double.class)) {
double v = getDoubleValue(section, fld.getName(), (double) defaultFldValue, comment);
fld.set(obj, v);
} else if (fld.getType().equals(byte.class)) {
byte v = (byte) getLongValue(section, fld.getName(), (long) (byte) defaultFldValue,
comment);
fld.set(obj, v);
} else if (fld.getType().equals(char.class)) {
char v = (char) getLongValue(section, fld.getName(), (long) (char) defaultFldValue,
comment);
fld.set(obj, v);
} else if (fld.getType().equals(short.class)) {
short v = (short) getLongValue(section, fld.getName(), (long) (short) defaultFldValue,
comment);
fld.set(obj, v);
} else if (fld.getType().equals(int.class)) {
int v = (int) getLongValue(section, fld.getName(), (long) (int) defaultFldValue,
comment);
fld.set(obj, v);
} else if (fld.getType().equals(long.class)) {
long v = getLongValue(section, fld.getName(), (long) defaultFldValue, comment);
fld.set(obj, v);
} else {
throw new RuntimeException("unhandled type " + fld.getType().getName());
}
} else if (fld.getType().equals(String.class)) {
String v = getStringValue(section, fld.getName(), (String) defaultFldValue, comment);
fld.set(obj, v);
} else if (fld.getType().isEnum()) {
String sv = getStringValue(section, fld.getName(), defaultFldValue.toString(), comment);
@SuppressWarnings({"unchecked", "rawtypes"})
Object v = Enum.valueOf((Class<Enum>) fld.getType(), sv);
fld.set(obj, v);
// TODO: this is a hack and needs to be resolved later
} else if (fld.getType().getName().equals("org.ray.api.UniqueID")) {
String sv = getStringValue(section, fld.getName(), defaultFldValue.toString(), comment);
Object v;
try {
v = fld.getType().getConstructor(new Class<?>[]{String.class}).newInstance(sv);
} catch (NoSuchMethodException | SecurityException | InstantiationException | InvocationTargetException e) {
System.err.println(
section + "." + fld.getName() + "'s format (" + sv + ") is invalid, default to "
+ defaultFldValue.toString());
v = defaultFldValue;
}
fld.set(obj, v);
} else if (fld.getType().isArray()) {
Class<?> ccls = fld.getType().getComponentType();
String ss = getStringValue(section, fld.getName(), null, comment);
if (null == ss) {
fld.set(obj, defaultFldValue);
} else {
Vector<String> ls = StringUtil.Split(ss, splitters, "", "");
if (ccls.equals(boolean.class)) {
boolean[] v = ObjectUtil
.toBooleanArray(ls.stream().map(Boolean::parseBoolean).toArray());
fld.set(obj, v);
} else if (ccls.equals(double.class)) {
double[] v = ls.stream().mapToDouble(Double::parseDouble).toArray();
fld.set(obj, v);
} else if (ccls.equals(int.class)) {
int[] v = ls.stream().mapToInt(Integer::parseInt).toArray();
fld.set(obj, v);
} else if (ccls.equals(long.class)) {
long[] v = ls.stream().mapToLong(Long::parseLong).toArray();
fld.set(obj, v);
} else if (ccls.equals(String.class)) {
String[] v;
if (StringUtil.isNullOrEmpty(defaultArrayIndirectSectionName)) {
v = ls.toArray(new String[]{});
} else {
v = this
.getIndirectStringArray(section, fld.getName(),
defaultArrayIndirectSectionName,
comment);
}
fld.set(obj, v);
} else {
throw new RuntimeException(
"Array with component type " + ccls.getName() + " is not supported yet");
}
}
} else {
Object fldObj = ObjectUtil.newObject(fld.getType());
fld.set(obj, fldObj);
readObject(section + "." + fld.getName(), fldObj, defaultFldValue);
}
} catch (IllegalArgumentException | IllegalAccessException e) {
throw new RuntimeException("set fld " + fld.getName() + " failed, err = " + e.getMessage(),
e);
}
}
}
}
java/common/src/main/java/org/ray/util/config/ConfigSection.java
+14
View File
@@ -0,0 +1,14 @@
package org.ray.util.config;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* A configuration section of related items.
*/
public class ConfigSection {
public String sectionKey;
public final Map<String, ConfigItem<?>> itemMap = new ConcurrentHashMap<>();
}
java/common/src/main/java/org/ray/util/config/CurrentUseConfig.java
+16
View File
@@ -0,0 +1,16 @@
package org.ray.util.config;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* The configuration which is currently in use.
*/
public class CurrentUseConfig {
public String filePath;
public final Map<String, ConfigSection> sectionMap = new ConcurrentHashMap<>();
}
java/common/src/main/java/org/ray/util/exception/TaskExecutionException.java
+15
View File
@@ -0,0 +1,15 @@
package org.ray.util.exception;
/**
* An exception which is thrown when a ray task encounters an error when executing.
*/
public class TaskExecutionException extends RuntimeException {
public TaskExecutionException(Throwable cause) {
super(cause);
}
public TaskExecutionException(String message, Throwable cause) {
super(message, cause);
}
}
java/common/src/main/java/org/ray/util/generator/Composition.java
+38
View File
@@ -0,0 +1,38 @@
package org.ray.util.generator;
import java.util.ArrayList;
import java.util.List;
/**
* Calculate all compositions for Parameter's count + Return's count, this class is used by code
* generators.
*/
public class Composition {
public static class TR {
public final int Tcount;
public final int Rcount;
public TR(int tcount, int rcount) {
super();
Tcount = tcount;
Rcount = rcount;
}
}
public static List<TR> calculate(int maxT, int maxR) {
List<TR> ret = new ArrayList<>();
for (int t = 0; t <= maxT; t++) {
// <= 0 for dynamic return count
// 0 for call_n returns RayMap<RID, x>
// -1 for call_n returns RayObject<>[N]
for (int r = -1; r <= maxR; r++) {
ret.add(new TR(t, r));
}
}
return ret;
}
}
java/common/src/main/java/org/ray/util/generator/FuncsGenerator.java
+87
View File
@@ -0,0 +1,87 @@
package org.ray.util.generator;
import java.io.IOException;
import org.ray.util.FileUtil;
import org.ray.util.generator.Composition.TR;
/**
* Generate all classes in org.ray.api.funcs
*/
public class FuncsGenerator {
public static void main(String[] args) throws IOException {
String rootdir = System.getProperty("user.dir") + "/../api/src/main/java";
rootdir += "/org/ray/api/funcs";
generate(rootdir);
}
private static void generate(String rootdir) throws IOException {
for (TR tr : Composition.calculate(Share.MAX_T, Share.MAX_R)) {
String str = build(tr.Tcount, tr.Rcount);
String file = rootdir + "/RayFunc_" + tr.Tcount + "_"
+ (tr.Rcount <= 0 ? (tr.Rcount == 0 ? "n" : "n_list") : tr.Rcount) + ".java";
FileUtil.overrideFile(file, str);
System.err.println("override " + file);
}
}
/*
* package org.ray.api.funcs;
*
* @FunctionalInterface
* public interface RayFunc_4_1<T0, T1, T2, T3, R0> extends RayFunc { R0
* apply();
*
* public static <R0> R0 execute(Object[] args) throws Throwable { String name =
* (String)args[args.length - 2]; assert (name.equals(RayFunc_0_1.class.getName())); byte[]
* funcBytes = (byte[])args[args.length - 1]; RayFunc_0_1<R0> f = (RayFunc_0_1<R0>)SerializationUtils.deserialize(funcBytes);
* return f.apply(); } }
*/
private static String build(int Tcount, int Rcount) {
StringBuilder sb = new StringBuilder();
String tname =
"Ray" + "Func_" + Tcount + "_" + (Rcount <= 0 ? (Rcount == 0 ? "n" : "n_list") : Rcount);
String gname = tname + "<" + Share.buildClassDeclare(Tcount, Rcount) + ">";
sb.append("package org.ray.api.funcs;").append("\n");
if (Rcount > 1) {
sb.append("import org.ray.api.returns.*;").append("\n");
}
if (Rcount <= 0) {
sb.append("import java.util.Collection;").append("\n");
sb.append("import java.util.List;").append("\n");
sb.append("import java.util.Map;").append("\n");
}
sb.append("import org.ray.api.*;").append("\n");
sb.append("import org.ray.api.internal.*;").append("\n");
sb.append("import org.apache.commons.lang3.SerializationUtils;").append("\n");
sb.append("\n");
sb.append("@FunctionalInterface").append("\n");
sb.append("public interface ").append(gname).append(" extends RayFunc {")
.append("\n");
sb.append("\t").append(Share.buildFuncReturn(Rcount)).append(" apply(")
.append(Rcount == 0 ? ("Collection<RID> returnids" + (Tcount > 0 ? ", " : "")) : "")
.append(Share.buildParameter(Tcount, "T", null)).append(") throws Throwable;")
.append("\n");
sb.append("\t\n");
sb.append("\tpublic static " + "<").append(Share.buildClassDeclare(Tcount, Rcount))
.append(">").append(" ").append(Share.buildFuncReturn(Rcount))
.append(" execute(Object[] args) throws Throwable {").append("\n");
sb.append("\t\tString name = (String)args[args.length - 2];").append("\n");
sb.append("\t\tassert (name.equals(").append(tname).append(".class.getName()));").append("\n");
sb.append("\t\tbyte[] funcBytes = (byte[])args[args.length - 1];").append("\n");
sb.append("\t\t").append(gname).append(" f = SerializationUtils.deserialize(funcBytes);")
.append("\n");
sb.append("\t\treturn f.apply(")
.append(Rcount == 0 ? ("(Collection<RID>)args[0]" + (Tcount > 0 ? ", " : "")) : "")
.append(Share.buildParameterUse2(Tcount, Rcount == 0 ? 1 : 0, "T", "args[", "]"))
.append(");").append("\n");
sb.append("\t}").append("\n");
sb.append("\t\n");
sb.append("}").append("\n");
return sb.toString();
}
}
java/common/src/main/java/org/ray/util/generator/MultipleReturnGenerator.java
+68
View File
@@ -0,0 +1,68 @@
package org.ray.util.generator;
import java.io.IOException;
import org.ray.util.FileUtil;
/**
* Generate all classes in org.ray.api.returns.MultipleReturnsX
*/
public class MultipleReturnGenerator {
public static void main(String[] args) throws IOException {
String rootdir = System.getProperty("user.dir") + "/../api/src/main/java";
rootdir += "/org/ray/api/returns";
for (int r = 2; r <= Share.MAX_R; r++) {
String str = build(r);
String file = rootdir + "/MultipleReturns" + r + ".java";
FileUtil.overrideFile(file, str);
System.err.println("override " + file);
}
}
/**
* package org.ray.api.returns;
*
* public class MultipleReturns2<R0, R1> extends MultipleReturns {
*
* public MultipleReturns2(R0 r0, R1 r1) { super(new Object[] { r0, r1 }); }
*
* public R0 get0() { return (R0) this.values[0]; }
*
* public R1 get1() { return (R1) this.values[1]; }
*
* }
*/
private static String build(int Rcount) {
StringBuilder sb = new StringBuilder();
sb.append("package org.ray.api.returns;").append("\n");
sb.append("import org.ray.api.*;").append("\n");
sb.append("@SuppressWarnings(\"unchecked\")");
sb.append("public class MultipleReturns").append(Rcount).append("<")
.append(Share.buildClassDeclare(0, Rcount)).append("> extends MultipleReturns {")
.append("\n");
sb.append("\tpublic MultipleReturns").append(Rcount).append("(")
.append(Share.buildParameter(Rcount, "R", null)).append(") {")
.append("\n");
sb.append("\t\tsuper(new Object[] { ").append(Share.buildParameterUse(Rcount, "R"))
.append(" });")
.append("\n");
sb.append("\t}").append("\n");
for (int k = 0; k < Rcount; k++) {
sb.append(buildGetter(k));
}
sb.append("}").append("\n");
return sb.toString();
}
/*
* @SuppressWarnings("unchecked") public R1 get1() { return (R1) this.values[1]; }
*/
private static String buildGetter(int index) {
return ("\tpublic R" + index + " get" + index + "() {\n")
+ "\t\treturn (R" + index + ") this.values[" + index + "];\n"
+ "\t}\n";
}
}
java/common/src/main/java/org/ray/util/generator/RayObjectsGenerator.java
+71
View File
@@ -0,0 +1,71 @@
package org.ray.util.generator;
import java.io.IOException;
import org.ray.util.FileUtil;
/**
* Generate all classes in org.ray.api.returns.RayObjectsX
*/
public class RayObjectsGenerator {
public static void main(String[] args) throws IOException {
String rootdir = System.getProperty("user.dir") + "/../api/src/main/java";
rootdir += "/org/ray/api/returns";
for (int r = 2; r <= Share.MAX_R; r++) {
String str = build(r);
String file = rootdir + "/RayObjects" + r + ".java";
FileUtil.overrideFile(file, str);
System.err.println("override " + file);
}
}
/*
* package org.ray.api.returns;
*
* import org.ray.api.RayObject; import org.ray.spi.model.UniqueID;
*
* @SuppressWarnings({"rawtypes", "unchecked"}) public class RayObjects2<R0, R1> extends
* RayObjects {
*
* public RayObjects2(UniqueID[] ids) { super(ids); }
*
* public RayObjects2(RayObject objs[]) { super(objs); }
*
* public RayObject<R0> r0() { return objs[0]; }
*
* public RayObject<R1> r1() { return objs[1]; } }
*/
private static String build(int Rcount) {
StringBuilder sb = new StringBuilder();
sb.append("package org.ray.api.returns;\n");
sb.append("import org.ray.api.*;\n");
sb.append("import org.ray.spi.model.UniqueID;\n");
sb.append("@SuppressWarnings({\"rawtypes\", \"unchecked\"})");
sb.append("public class RayObjects").append(Rcount).append("<")
.append(Share.buildClassDeclare(0, Rcount)).append("> extends RayObjects {")
.append("\n");
sb.append("\tpublic RayObjects").append(Rcount).append("(UniqueID[] ids) {").append("\n");
sb.append("\t\tsuper(ids);").append("\n");
sb.append("\t}").append("\n");
sb.append("\tpublic RayObjects").append(Rcount).append("(RayObject objs[]) {").append("\n");
sb.append("\t\tsuper(objs);").append("\n");
sb.append("\t}").append("\n");
for (int k = 0; k < Rcount; k++) {
sb.append(buildGetter(k));
}
sb.append("}").append("\n");
return sb.toString();
}
/**
* public RayObject<R0> r0() { return objs[0]; }
*/
private static String buildGetter(int index) {
return "\tpublic RayObject<R" + index + "> r" + index + "() {\n"
+ "\t\treturn objs[" + index + "];\n"
+ "\t}\n";
}
}
java/common/src/main/java/org/ray/util/generator/RpcGenerator.java
+175
View File
@@ -0,0 +1,175 @@
package org.ray.util.generator;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import org.ray.util.FileUtil;
import org.ray.util.generator.Composition.TR;
/**
* Generate Rpc.java
*/
public class RpcGenerator {
public static void main(String[] args) throws IOException {
String rootdir = System.getProperty("user.dir") + "/../api/src/main/java";
rootdir += "/org/ray/api/Rpc.java";
FileUtil.overrideFile(rootdir, build());
}
private static String build() {
StringBuilder sb = new StringBuilder();
sb.append("package org.ray.api.internal;\n");
sb.append("import org.ray.api.funcs.*;\n");
sb.append("import org.ray.api.returns.*;\n");
sb.append("import org.ray.api.*;\n");
sb.append("import java.util.Collection;\n");
sb.append("import java.util.Map;\n");
sb.append("@SuppressWarnings({\"rawtypes\", \"unchecked\"})\n");
sb.append("class Rpc {\n");
for (TR tr : Composition.calculate(Share.MAX_T, Share.MAX_R)) {
buildCall(sb, tr.Tcount, tr.Rcount);
}
sb.append("}\n");
return sb.toString();
}
private static void buildCall(StringBuilder sb, int Tcount, int Rcount) {
for (Set<Integer> whichTisFuture : whichTisFutureComposition(Tcount)) {
sb.append(buildCall(Tcount, Rcount, whichTisFuture));
}
}
/**
* public static <T0, R0> RayObject<R0> call(RayFunc_1_1<T0, R0> f, RayObject<T0> arg) { return
* Ray.runtime().rpc(() -> f.apply(null), arg).objs[0]; }
*/
private static String buildCall(int Tcount, int Rcount, Set<Integer> whichTisFuture) {
StringBuilder sb = new StringBuilder();
String parameter = (Tcount == 0 ? ""
: ", " + Share.buildParameter(Tcount, "T", whichTisFuture));
sb.append("\tpublic static <").append(Share.buildClassDeclare(Tcount, Rcount)).append("> ")
.append(Share.buildRpcReturn(Rcount)).append(" call")
.append(Rcount == 1 ? "" : (Rcount <= 0 ? "_n" : ("_" + Rcount))).append("(RayFunc_")
.append(Tcount).append("_")
.append(Rcount <= 0 ? (Rcount == 0 ? "n" : "n_list") : Rcount).append("<")
.append(Share.buildClassDeclare(Tcount, Rcount)).append("> f").append(
Rcount <= 0 ? (Rcount == 0 ? ", Collection<RID> returnids" : ", Integer returnCount")
: "").append(parameter).append(") {\n");
/*
* public static <R0> RayObject<R0> call(RayFunc_0_1<R0> f) {
if (Ray.Parameters().remoteLambda()) {
return Ray.internal().call(RayFunc_0_1.class, f, 1).objs[0];
}
else {
return Ray.internal().call(() -> f.apply(), 1).objs[0];
}
}
*/
String nulls = Share.buildRepeat("null",
Tcount + (Rcount == 0 ? 1/*for first arg map*/ : 0));
String parameterUse = (Tcount == 0 ? "" : (", " + Share.buildParameterUse(Tcount, "T")));
String labmdaUse = "RayFunc_"
+ Tcount + "_" + (Rcount <= 0 ? (Rcount == 0 ? "n" : "n_list") : Rcount)
+ ".class, f";
sb.append("\t\tif (Ray.Parameters().remoteLambda()) {\n");
if (Rcount == 1) {
sb.append("\t\t\treturn Ray.internal().call(null, ").append(labmdaUse).append(", 1")
.append(parameterUse).append(").objs[0];")
.append("\n");
} else if (Rcount == 0) {
sb.append("\t\t\treturn Ray.internal().callWithReturnLabels(null, ")
.append(labmdaUse).append(", returnids").append(parameterUse).append(");")
.append("\n");
} else if (Rcount < 0) {
sb.append("\t\t\treturn Ray.internal().callWithReturnIndices(null, ")
.append(labmdaUse).append(", returnCount").append(parameterUse).append(");")
.append("\n");
} else {
sb.append("\t\t\treturn new RayObjects").append(Rcount)
.append("(Ray.internal().call(null, ").append(labmdaUse).append(", ").append(Rcount)
.append(parameterUse).append(").objs);")
.append("\n");
}
sb.append("\t\t} else {\n");
if (Rcount == 1) {
sb.append("\t\t\treturn Ray.internal().call(null, () -> f.apply(").append(nulls)
.append("), 1").append(parameterUse).append(").objs[0];")
.append("\n");
} else if (Rcount == 0) {
sb.append("\t\t\treturn Ray.internal().callWithReturnLabels(null, () -> f.apply(")
.append(nulls).append("), returnids").append(parameterUse).append(");")
.append("\n");
} else if (Rcount < 0) {
sb.append("\t\t\treturn Ray.internal().callWithReturnIndices(null, () -> f.apply(")
.append(nulls).append("), returnCount").append(parameterUse).append(");")
.append("\n");
} else {
sb.append("\t\t\treturn new RayObjects").append(Rcount)
.append("(Ray.internal().call(null, () -> f.apply(").append(nulls).append("), ")
.append(Rcount).append(parameterUse).append(").objs);")
.append("\n");
}
sb.append("\t\t}\n");
sb.append("\t}\n");
return sb.toString();
}
private static Set<Set<Integer>> whichTisFutureComposition(int Tcount) {
Set<Set<Integer>> ret = new HashSet<>();
Set<Integer> N = new HashSet<>();
for (int k = 0; k < Tcount; k++) {
N.add(k);
}
for (int k = 0; k <= Tcount; k++) {
ret.addAll(CNn(N, k));
}
return ret;
}
//pick n numbers in N
private static Set<Set<Integer>> CNn(Set<Integer> N, int n) {
C c = new C();
for (int k = 0; k < n; k++) {
c.mul(N);
}
return c.v;
}
static class C {
Set<Set<Integer>> v;
public C() {
v = new HashSet<>();
v.add(new HashSet<>());
}
void mul(Set<Integer> ns) {
Set<Set<Integer>> ret = new HashSet<>();
for (int n : ns) {
ret.addAll(mul(n));
}
this.v = ret;
}
Set<Set<Integer>> mul(int n) {
Set<Set<Integer>> ret = new HashSet<>();
for (Set<Integer> s : v) {
if (s.contains(n)) {
continue;
}
Set<Integer> ns = new HashSet<>(s);
ns.add(n);
ret.add(ns);
}
return ret;
}
}
}
java/common/src/main/java/org/ray/util/generator/Share.java
+139
View File
@@ -0,0 +1,139 @@
package org.ray.util.generator;
import java.util.Set;
/**
* Share util for generators
*/
public class Share {
public static final int MAX_T = 6;
public static final int MAX_R = 4;
/**
* T0, T1, T2, T3, R
*/
public static String buildClassDeclare(int Tcount, int Rcount) {
StringBuilder sb = new StringBuilder();
for (int k = 0; k < Tcount; k++) {
sb.append("T").append(k).append(", ");
}
if (Rcount == 0) {
sb.append("R, RID");
} else if (Rcount < 0) {
assert (Rcount == -1);
sb.append("R");
} else {
for (int k = 0; k < Rcount; k++) {
sb.append("R").append(k).append(", ");
}
sb.deleteCharAt(sb.length() - 1);
sb.deleteCharAt(sb.length() - 1);
}
return sb.toString();
}
/**
* T0 t0, T1 t1, T2 t2, T3 t3
*/
public static String buildParameter(int Tcount, String TR, Set<Integer> whichTisFuture) {
StringBuilder sb = new StringBuilder();
for (int k = 0; k < Tcount; k++) {
sb.append(whichTisFuture != null && whichTisFuture.contains(k)
? "RayObject<" + (TR + k) + ">" : (TR + k)).append(" ").append(TR.toLowerCase())
.append(k).append(", ");
}
if (Tcount > 0) {
sb.deleteCharAt(sb.length() - 1);
sb.deleteCharAt(sb.length() - 1);
}
return sb.toString();
}
/**
* t0, t1, t2
*/
public static String buildParameterUse(int Tcount, String TR) {
StringBuilder sb = new StringBuilder();
for (int k = 0; k < Tcount; k++) {
sb.append(TR.toLowerCase()).append(k).append(", ");
}
if (Tcount > 0) {
sb.deleteCharAt(sb.length() - 1);
sb.deleteCharAt(sb.length() - 1);
}
return sb.toString();
}
public static String buildParameterUse2(int Tcount, int startIndex, String TR, String pre,
String post) {
StringBuilder sb = new StringBuilder();
for (int k = 0; k < Tcount; k++) {
sb.append("(").append(TR).append(k).append(")").append(pre).append(k + startIndex)
.append(post).append(", ");
}
if (Tcount > 0) {
sb.deleteCharAt(sb.length() - 1);
sb.deleteCharAt(sb.length() - 1);
}
return sb.toString();
}
/**
* MultipleReturns2<R0, R1> apply(); R0
*/
public static String buildFuncReturn(int Rcount) {
if (Rcount == 0) {
return "Map<RID, R>";
} else if (Rcount < 0) {
assert (-1 == Rcount);
return "List<R>";
}
if (Rcount == 1) {
return "R0";
}
StringBuilder sb = new StringBuilder();
for (int k = 0; k < Rcount; k++) {
sb.append("R").append(k).append(", ");
}
sb.deleteCharAt(sb.length() - 1);
sb.deleteCharAt(sb.length() - 1);
return "MultipleReturns" + Rcount + "<" + sb.toString() + ">";
}
/**
*/
public static String buildRpcReturn(int Rcount) {
if (Rcount == 0) {
return "RayMap<RID, R>";
} else if (Rcount < 0) {
assert (Rcount == -1);
return "RayList<R>";
}
if (Rcount == 1) {
return "RayObject<R0>";
}
StringBuilder sb = new StringBuilder();
for (int k = 0; k < Rcount; k++) {
sb.append("R").append(k).append(", ");
}
sb.deleteCharAt(sb.length() - 1);
sb.deleteCharAt(sb.length() - 1);
return "RayObjects" + Rcount + "<" + sb.toString() + ">";
}
public static String buildRepeat(String toRepeat, int count) {
StringBuilder ret = new StringBuilder();
for (int k = 0; k < count; k++) {
ret.append(toRepeat).append(", ");
}
if (count > 0) {
ret.deleteCharAt(ret.length() - 1);
ret.deleteCharAt(ret.length() - 1);
}
return ret.toString();
}
}
java/common/src/main/java/org/ray/util/logger/ConsoleLogger.java
+46
View File
@@ -0,0 +1,46 @@
package org.ray.util.logger;
import org.apache.log4j.Logger;
/**
* A logger which prints output to console
*/
public class ConsoleLogger extends Logger {
final Logger realLogger;
protected ConsoleLogger(String name, Logger realLogger) {
super(name);
this.realLogger = realLogger;
}
@Override
public void info(Object log) {
realLogger.info("(" + this.getName() + ") " + log);
}
@Override
public void warn(Object log) {
realLogger.warn("(" + this.getName() + ") " + log);
}
@Override
public void warn(Object log, Throwable e) {
realLogger.warn("(" + this.getName() + ") " + log, e);
}
@Override
public void debug(Object log) {
realLogger.debug("(" + this.getName() + ") " + log);
}
@Override
public void error(Object log) {
realLogger.error("(" + this.getName() + ") " + log);
}
@Override
public void error(Object log, Throwable e) {
realLogger.error("(" + this.getName() + ") " + log, e);
}
}
java/common/src/main/java/org/ray/util/logger/DynamicLog.java
+260
View File
@@ -0,0 +1,260 @@
package org.ray.util.logger;
import java.util.Collection;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.log4j.Logger;
import org.ray.util.CommonUtil;
/**
* Dynamic logger without properties configuration file
*/
public class DynamicLog {
static final ThreadLocal<String> PREFIX = new ThreadLocal<>();
private static LogLevel logLevel = LogLevel.DEBUG;
private static Boolean logLevelSetFlag = false;
/**
* set the context prefix for all logs
*/
public static void setContextPrefix(String prefix) {
PREFIX.set(prefix);
}
public static String getContextPrefix() {
return PREFIX.get();
}
/**
* set the level for all logs
*/
public static void setLogLevel(String level) {
if (logLevelSetFlag) { /* one shot, avoid the risk of multithreading */
return;
}
logLevelSetFlag = true;
logLevel = LogLevel.of(level);
}
private static LogLevel getenumLogLevel() {
return logLevel;
}
@Override
public String toString() {
return this.getKey();
}
@Override
public boolean equals(Object o) {
return this.toString().equals(o.toString());
}
@Override
public int hashCode() {
return this.toString().hashCode();
}
private String wrap(String level, String log) {
StackTraceElement stes[] = Thread.currentThread().getStackTrace();
String ret = "[" + level + "]" + "[" + stes[3].getFileName() + ":" + stes[3].getLineNumber()
+ "] - " + (log == null ? "" : log);
String prefix = PREFIX.get();
if (prefix != null) {
ret = "[" + prefix + "]" + ret;
}
return ret;
}
public void debug(String log) {
if (!getenumLogLevel().needLog(LogLevel.DEBUG)) {
return;
}
log = wrap("debug", log);
Logger loggers[] = DynamicLogManager.getLogs(this);
for (Logger logger : loggers) {
logger.debug(log);
}
}
public void info(String log) {
if (!getenumLogLevel().needLog(LogLevel.INFO)) {
return;
}
log = wrap("info", log);
Logger loggers[] = DynamicLogManager.getLogs(this);
for (Logger logger : loggers) {
logger.info(log);
}
}
public void warn(String log) {
if (!getenumLogLevel().needLog(LogLevel.WARN)) {
return;
}
log = wrap("warn", log);
Logger loggers[] = DynamicLogManager.getLogs(this);
for (Logger logger : loggers) {
logger.warn(log);
}
}
public void warn(String log, Throwable e) {
if (!getenumLogLevel().needLog(LogLevel.WARN)) {
return;
}
log = wrap("warn", log);
Logger loggers[] = DynamicLogManager.getLogs(this);
for (Logger logger : loggers) {
logger.warn(log, e);
}
}
public void error(String log) {
if (!getenumLogLevel().needLog(LogLevel.ERROR)) {
return;
}
log = wrap("error", log);
Logger loggers[] = DynamicLogManager.getLogs(this);
for (Logger logger : loggers) {
logger.error(log);
}
}
public void error(String log, Throwable e) {
if (!getenumLogLevel().needLog(LogLevel.ERROR)) {
return;
}
log = wrap("error", log);
if (e == null) {
error(log);
return;
}
Logger loggers[] = DynamicLogManager.getLogs(this);
for (Logger logger : loggers) {
logger.error(log, e);
}
}
public void error(Throwable e) {
if (!getenumLogLevel().needLog(LogLevel.ERROR)) {
return;
}
String log = wrap("error", e == null ? null : e.getMessage());
if (e == null) {
error(log);
return;
}
Logger loggers[] = DynamicLogManager.getLogs(this);
for (Logger logger : loggers) {
logger.error(log, e);
}
}
//statistic for sampling
private static class SampleStatis {
int total;
public boolean gamble() {
int randomRange;
if (total < 100) {
randomRange = 1;
} else if (total < 1000) {
randomRange = 1000;
} else if (total < 100000) {
randomRange = 10000;
} else if (total < 1000000) {
randomRange = 100000;
} else {
total = 0;
randomRange = 1;
}
if (CommonUtil.getRandom(randomRange) == 0) {
total++;
return true;
} else {
total++;
return false;
}
}
}
private static Map<String/*Samplekey*/, SampleStatis> sampleStatis = new ConcurrentHashMap<>();
/**
* Print sample error log
*/
public boolean sampleError(Object sampleKeyO, String log, Throwable e) {
String sampleKey = sampleKeyO.toString();
try {
SampleStatis ss = sampleStatis.computeIfAbsent(sampleKey, k -> new SampleStatis());
if (ss.gamble()) {
Logger loggers[] = DynamicLogManager.getLogs(this);
for (Logger logger : loggers) {
if (e != null) {
logger.error("[" + sampleKey + "] - " + log, e);
} else {
logger.error("[" + sampleKey + "] - " + log);
}
}
return true;
} else {
return false;
}
} finally {
if (sampleStatis.size() > 100000) {
sampleStatis = new ConcurrentHashMap<>();
}
}
}
private final String key;
private DynamicLog(String key) {
this.key = key;
}
public String getKey() {
return this.key;
}
public String getDefaultLogFileName() {
return this.key + ".log";
}
public static DynamicLog registerName(String name) {
return DynamicLogNameRegister.registerName(name);
}
public static Collection<DynamicLog> values() {
return DynamicLogNameRegister.names.values();
}
public static class DynamicLogNameRegister {
static final Map<String, DynamicLog> names = new ConcurrentHashMap<>();
public static DynamicLog registerName(String name) {
DynamicLog ret = names.get(name);
if (ret != null) {
return ret;
}
synchronized (names) {
ret = names.get(name);
if (ret != null) {
return ret;
}
ret = new DynamicLog(name);
names.put(name, ret);
return ret;
}
}
}
}
java/common/src/main/java/org/ray/util/logger/DynamicLogManager.java
+161
View File
@@ -0,0 +1,161 @@
package org.ray.util.logger;
import java.io.File;
import java.io.IOException;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout;
import org.apache.log4j.RollingFileAppender;
import org.ray.util.SystemUtil;
/**
* Manager for dynamic loggers
*/
public class DynamicLogManager {
//whether to print the log on std(ie. console)
public static boolean printOnStd = false;
//the root directory of log files
public static String logsDir;
public static String logsSuffix;
public static Level level = Level.DEBUG; //Level.INFO;
/** */
private static final int LOG_CACHE_SIZE = 32 * 1024;
protected final static String DAY_DATE_PATTERN = "'.'yyyy-MM-dd";
// private final static String HOUR_DATE_PATTERN = "'.'yyyy-MM-dd_HH";
// private final static String GBK = "GBK";
private final static String DAILY_APPENDER_NAME = "_DAILY_APPENDER_NAME";
// private final static String CONSOLE_APPENDER_NAME = "_CONSOLE_APPENDER_NAME";
private final static String LAYOUT_PATTERN = "%d [%t]%m%n";
private static final ConcurrentHashMap<DynamicLog, Logger> loggers = new ConcurrentHashMap<>();
private static int MAX_FILE_NUM = 10;
private static String MAX_FILE_SIZE = "500MB";
private static boolean initFinished = false;
/**
* init from system properties
* -DlogOutput=console/file_path
* if file_path contains *pid*, it will be replaced with real PID of this JAVA process
* if file_path contains *pid_suffix*, all log file will append the suffix -> xxx-pid.log
*/
static {
String logOutput = System.getProperty("logOutput");
if (null == logOutput
|| logOutput.equalsIgnoreCase("console")
|| logOutput.equalsIgnoreCase("std")
|| logOutput.equals("")) {
DynamicLogManager.printOnStd = true;
System.out.println("config log output as std");
} else {
if (logOutput.contains("*pid*")) {
logOutput = logOutput.replaceAll("\\*pid\\*", String.valueOf(SystemUtil.pid()));
}
if (logOutput.contains("*pid_suffix*")) {
logOutput = logOutput.replaceAll("\\*pid_suffix\\*", "");
if (logOutput.endsWith("/")) {
logOutput = logOutput.substring(0, logOutput.length() - 1);
}
DynamicLogManager.logsSuffix = String.valueOf(SystemUtil.pid());
}
System.out.println("config log output as " + logOutput);
DynamicLogManager.logsDir = logOutput;
}
String logLevel = System.getProperty("logLevel");
if (logLevel != null && logLevel.equals("debug")) {
level = Level.DEBUG;
}
}
public static synchronized void init(int maxFileNum, String maxFileSize) {
if (initFinished) {
return;
}
initFinished = true;
System.out.println(
"DynamicLogManager init with maxFileNum:" + maxFileNum + " maxFileSize:" + maxFileSize);
if (loggers.size() > 0) {
System.err
.println("already have logger be maked before init log file system, please check it");
}
MAX_FILE_NUM = maxFileNum;
MAX_FILE_SIZE = maxFileSize;
}
public static Logger[] getLogs(DynamicLog dynLog) {
Logger logger = loggers.get(dynLog);
if (logger == null) {
synchronized (loggers) {
logger = loggers.get(dynLog);
if (logger == null) {
logger = initLogger(dynLog);
}
}
}
return new Logger[]{logger};
}
private static Logger initLogger(DynamicLog dynLog) {
if (printOnStd) {
Logger reallogger = Logger.getLogger(dynLog.getKey());
ConsoleLogger logger = new ConsoleLogger(dynLog.getKey(), reallogger);
PatternLayout layout = new PatternLayout(LAYOUT_PATTERN);
ConsoleAppender appender = new ConsoleAppender(layout, ConsoleAppender.SYSTEM_OUT);
reallogger.removeAllAppenders();
reallogger.addAppender(appender);
reallogger.setLevel(level);
reallogger.setAdditivity(false);
loggers.putIfAbsent(dynLog, logger);
return logger;
} else {
Logger logger = makeLogger(dynLog.getKey(), dynLog.getDefaultLogFileName());
loggers.putIfAbsent(dynLog, logger);
return logger;
}
}
protected static Logger makeLogger(String loggerName, String filename) {
Logger logger = Logger.getLogger(loggerName);
PatternLayout layout = new PatternLayout(LAYOUT_PATTERN);
File dir = new File(logsDir);
if (!dir.exists()) {
dir.mkdirs();
}
String logFileName = logsDir + "/" + filename;
if (logsSuffix != null) {
logFileName = logFileName.substring(0, logFileName.length() - 4) + "-" + logsSuffix
+ ".log";
}
System.out.println("new_log_path:" + logFileName);
RollingFileAppender appender;
try {
appender = new TimedFlushDailyRollingFileAppender(layout, logFileName);
appender.setAppend(true);
appender.setEncoding("UTF-8");
appender.setName(DAILY_APPENDER_NAME);
appender.setBufferSize(LOG_CACHE_SIZE);
appender.setBufferedIO(true);
appender.setImmediateFlush(false);
appender.setMaxBackupIndex(MAX_FILE_NUM);
appender.setMaxFileSize(MAX_FILE_SIZE);
appender.activateOptions();
} catch (IOException e) {
throw new RuntimeException(e);
}
logger.removeAllAppenders();
logger.addAppender(appender);
logger.setLevel(level);
logger.setAdditivity(false);
return logger;
}
}
java/common/src/main/java/org/ray/util/logger/LogLevel.java
+29
View File
@@ -0,0 +1,29 @@
package org.ray.util.logger;
public enum LogLevel {
ERROR("error", 0),
WARN("warn", 1),
INFO("info", 2),
DEBUG("debug", 3);
private final String name;
private final int index;
LogLevel(String name, int index) {
this.name = name;
this.index = index;
}
public static LogLevel of(String name) {
for (LogLevel level : values()) {
if (level.name.equals(name)) {
return level;
}
}
return null;
}
public Boolean needLog(LogLevel level) {
return level.index <= this.index;
}
}
java/common/src/main/java/org/ray/util/logger/RayLog.java
+17
View File
@@ -0,0 +1,17 @@
package org.ray.util.logger;
/**
* Dynamic loggers in Ray
*/
public class RayLog {
/**
* for ray itself
*/
public static final DynamicLog core = DynamicLog.registerName("core");
/**
* for ray's app's log
*/
public static DynamicLog rapp = core; //DynamicLog.registerName("rapp");
}
java/common/src/main/java/org/ray/util/logger/TimedFlushDailyRollingFileAppender.java
+68
View File
@@ -0,0 +1,68 @@
package org.ray.util.logger;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import org.apache.log4j.Layout;
import org.apache.log4j.RollingFileAppender;
/**
* Normal log appender
*/
public class TimedFlushDailyRollingFileAppender extends RollingFileAppender {
private final static Set<TimedFlushDailyRollingFileAppender> all = new HashSet<>();
public TimedFlushDailyRollingFileAppender() {
super();
synchronized (all) {
all.add(this);
}
}
public TimedFlushDailyRollingFileAppender(Layout layout, String filename) throws IOException {
super(layout, filename);
synchronized (all) {
all.add(this);
}
}
static {
new TimedFlushLogThread().start();
}
private void flush() {
try {
if (!checkEntryConditions()) {
return;
}
qw.flush();
} catch (Exception e) {
e.printStackTrace();
}
}
private static class TimedFlushLogThread extends Thread {
public TimedFlushLogThread() {
super();
setName("TimedFlushLogThread");
setDaemon(true);
}
public void run() {
while (true) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
synchronized (all) {
for (TimedFlushDailyRollingFileAppender appender : all) {
appender.flush();
}
}
}
}
}
}