wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-18 12:40:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

update to slf4j, remove DynamicLog (#2384)

Browse Source
This commit is contained in:
Hanwei Jin
2018-07-09 23:33:59 -07:00
committed by Philipp Moritz
parent 55d5e28872
commit 450b11f1d6
15 changed files with 70 additions and 613 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/api/src/main/java/org/ray/api/Ray.java
-8
View File
@@ -3,7 +3,6 @@ package org.ray.api;
import java.util.List;
import org.ray.api.internal.RayConnector;
import org.ray.util.exception.TaskExecutionException;
import org.ray.util.logger.DynamicLog;
import org.ray.util.logger.RayLog;
/**
@@ -102,11 +101,4 @@ public final class Ray extends Rpc {
static RayApi internal() {
return impl;
}
/**
* for ray's app's log.
*/
public static DynamicLog getRappLogger() {
return RayLog.rapp;
}
}
java/common/pom.xml
+2 -2
View File
@@ -20,8 +20,8 @@
<dependencies>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
<groupId>quartz</groupId>
java/common/src/main/java/org/ray/util/logger/ConsoleLogger.java
-46
View File
@@ -1,46 +0,0 @@
package org.ray.util.logger;
import org.apache.log4j.Logger;
/**
* A logger which prints output to console.
*/
public class ConsoleLogger extends Logger {
final Logger realLogger;
protected ConsoleLogger(String name, Logger realLogger) {
super(name);
this.realLogger = realLogger;
}
@Override
public void debug(Object log) {
realLogger.debug("(" + this.getName() + ") " + log);
}
@Override
public void error(Object log) {
realLogger.error("(" + this.getName() + ") " + log);
}
@Override
public void error(Object log, Throwable e) {
realLogger.error("(" + this.getName() + ") " + log, e);
}
@Override
public void info(Object log) {
realLogger.info("(" + this.getName() + ") " + log);
}
@Override
public void warn(Object log) {
realLogger.warn("(" + this.getName() + ") " + log);
}
@Override
public void warn(Object log, Throwable e) {
realLogger.warn("(" + this.getName() + ") " + log, e);
}
}
java/common/src/main/java/org/ray/util/logger/DynamicLog.java
-258
View File
@@ -1,258 +0,0 @@
package org.ray.util.logger;
import java.util.Collection;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.log4j.Logger;
import org.ray.util.CommonUtil;
/**
* Dynamic logger without properties configuration file.
*/
public class DynamicLog {
static final ThreadLocal<String> PREFIX = new ThreadLocal<>();
private static LogLevel logLevel = LogLevel.DEBUG;
private static Boolean logLevelSetFlag = false;
private static Map<String/*Samplekey*/, SampleStatis> sampleStatis = new ConcurrentHashMap<>();
private final String key;
private DynamicLog(String key) {
this.key = key;
}
public static String getContextPrefix() {
return PREFIX.get();
}
/**
* set the context prefix for all logs.
*/
public static void setContextPrefix(String prefix) {
PREFIX.set(prefix);
}
/**
* set the level for all logs.
*/
public static void setLogLevel(String level) {
if (logLevelSetFlag) { /* one shot, avoid the risk of multithreading */
return;
}
logLevelSetFlag = true;
logLevel = LogLevel.of(level);
}
public static DynamicLog registerName(String name) {
return DynamicLogNameRegister.registerName(name);
}
public static Collection<DynamicLog> values() {
return DynamicLogNameRegister.names.values();
}
@Override
public int hashCode() {
return this.toString().hashCode();
}
@Override
public boolean equals(Object o) {
return this.toString().equals(o.toString());
}
@Override
public String toString() {
return this.getKey();
}
public String getKey() {
return this.key;
}
public void debug(String log) {
if (!getenumLogLevel().needLog(LogLevel.DEBUG)) {
return;
}
log = wrap("debug", log);
Logger[] loggers = DynamicLogManager.getLogs(this);
for (Logger logger : loggers) {
logger.debug(log);
}
}
private static LogLevel getenumLogLevel() {
return logLevel;
}
private String wrap(String level, String log) {
StackTraceElement[] stes = Thread.currentThread().getStackTrace();
String ret = "[" + level + "]" + "[" + stes[3].getFileName() + ":" + stes[3].getLineNumber()
+ "] - " + (log == null ? "" : log);
String prefix = PREFIX.get();
if (prefix != null) {
ret = "[" + prefix + "]" + ret;
}
return ret;
}
public void info(String log) {
if (!getenumLogLevel().needLog(LogLevel.INFO)) {
return;
}
log = wrap("info", log);
Logger[] loggers = DynamicLogManager.getLogs(this);
for (Logger logger : loggers) {
logger.info(log);
}
}
public void warn(String log) {
if (!getenumLogLevel().needLog(LogLevel.WARN)) {
return;
}
log = wrap("warn", log);
Logger[] loggers = DynamicLogManager.getLogs(this);
for (Logger logger : loggers) {
logger.warn(log);
}
}
public void warn(String log, Throwable e) {
if (!getenumLogLevel().needLog(LogLevel.WARN)) {
return;
}
log = wrap("warn", log);
Logger[] loggers = DynamicLogManager.getLogs(this);
for (Logger logger : loggers) {
logger.warn(log, e);
}
}
public void error(String log, Throwable e) {
if (!getenumLogLevel().needLog(LogLevel.ERROR)) {
return;
}
log = wrap("error", log);
if (e == null) {
error(log);
return;
}
Logger[] loggers = DynamicLogManager.getLogs(this);
for (Logger logger : loggers) {
logger.error(log, e);
}
}
public void error(String log) {
if (!getenumLogLevel().needLog(LogLevel.ERROR)) {
return;
}
log = wrap("error", log);
Logger[] loggers = DynamicLogManager.getLogs(this);
for (Logger logger : loggers) {
logger.error(log);
}
}
public void error(Throwable e) {
if (!getenumLogLevel().needLog(LogLevel.ERROR)) {
return;
}
String log = wrap("error", e == null ? null : e.getMessage());
if (e == null) {
error(log);
return;
}
Logger[] loggers = DynamicLogManager.getLogs(this);
for (Logger logger : loggers) {
logger.error(log, e);
}
}
/**
* Print sample error log.
*/
public boolean sampleError(Object sampleKeyO, String log, Throwable e) {
String sampleKey = sampleKeyO.toString();
try {
SampleStatis ss = sampleStatis.computeIfAbsent(sampleKey, k -> new SampleStatis());
if (ss.gamble()) {
Logger[] loggers = DynamicLogManager.getLogs(this);
for (Logger logger : loggers) {
if (e != null) {
logger.error("[" + sampleKey + "] - " + log, e);
} else {
logger.error("[" + sampleKey + "] - " + log);
}
}
return true;
} else {
return false;
}
} finally {
if (sampleStatis.size() > 100000) {
sampleStatis = new ConcurrentHashMap<>();
}
}
}
public String getDefaultLogFileName() {
return this.key + ".log";
}
//statistic for sampling
private static class SampleStatis {
int total;
public boolean gamble() {
int randomRange;
if (total < 100) {
randomRange = 1;
} else if (total < 1000) {
randomRange = 1000;
} else if (total < 100000) {
randomRange = 10000;
} else if (total < 1000000) {
randomRange = 100000;
} else {
total = 0;
randomRange = 1;
}
if (CommonUtil.getRandom(randomRange) == 0) {
total++;
return true;
} else {
total++;
return false;
}
}
}
public static class DynamicLogNameRegister {
static final Map<String, DynamicLog> names = new ConcurrentHashMap<>();
public static DynamicLog registerName(String name) {
DynamicLog ret = names.get(name);
if (ret != null) {
return ret;
}
synchronized (names) {
ret = names.get(name);
if (ret != null) {
return ret;
}
ret = new DynamicLog(name);
names.put(name, ret);
return ret;
}
}
}
}
java/common/src/main/java/org/ray/util/logger/DynamicLogManager.java
-160
View File
@@ -1,160 +0,0 @@
package org.ray.util.logger;
import java.io.File;
import java.io.IOException;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout;
import org.apache.log4j.RollingFileAppender;
import org.ray.util.SystemUtil;
/**
* Manager for dynamic loggers.
*/
public class DynamicLogManager {
protected static final String DAY_DATE_PATTERN = "'.'yyyy-MM-dd";
private static final int LOG_CACHE_SIZE = 32 * 1024;
// private final static String HOUR_DATE_PATTERN = "'
// .'yyyy-MM-dd_HH";
// private final static String GBK = "GBK";
private static final String DAILY_APPENDER_NAME = "_DAILY_APPENDER_NAME";
// private final static String CONSOLE_APPENDER_NAME =
// "_CONSOLE_APPENDER_NAME";
private static final String LAYOUT_PATTERN = "%d [%t]%m%n";
private static final ConcurrentHashMap<DynamicLog, Logger> loggers = new ConcurrentHashMap<>();
//whether to print the log on std(ie. console)
public static boolean printOnStd = false;
//the root directory of log files
public static String logsDir;
public static String logsSuffix;
public static Level level = Level.DEBUG; //Level.INFO;
private static int MAX_FILE_NUM = 10;
private static String MAX_FILE_SIZE = "500MB";
private static boolean initFinished = false;
/**
* init from system properties
* -DlogOutput=console/file_path
* if file_path contains *pid*, it will be replaced with real PID of this JAVA process
* if file_path contains *pid_suffix*, all log file will append the suffix -> xxx-pid.log
*/
static {
String logOutput = System.getProperty("logOutput");
if (null == logOutput
|| logOutput.equalsIgnoreCase("console")
|| logOutput.equalsIgnoreCase("std")
|| logOutput.equals("")) {
DynamicLogManager.printOnStd = true;
System.out.println("config log output as std");
} else {
if (logOutput.contains("*pid*")) {
logOutput = logOutput.replaceAll("\\*pid\\*", String.valueOf(SystemUtil.pid()));
}
if (logOutput.contains("*pid_suffix*")) {
logOutput = logOutput.replaceAll("\\*pid_suffix\\*", "");
if (logOutput.endsWith("/")) {
logOutput = logOutput.substring(0, logOutput.length() - 1);
}
DynamicLogManager.logsSuffix = String.valueOf(SystemUtil.pid());
}
System.out.println("config log output as " + logOutput);
DynamicLogManager.logsDir = logOutput;
}
String logLevel = System.getProperty("logLevel");
if (logLevel != null && logLevel.equals("debug")) {
level = Level.DEBUG;
}
}
public static synchronized void init(int maxFileNum, String maxFileSize) {
if (initFinished) {
return;
}
initFinished = true;
System.out.println(
"DynamicLogManager init with maxFileNum:" + maxFileNum + " maxFileSize:" + maxFileSize);
if (loggers.size() > 0) {
System.err
.println("already have logger be maked before init log file system, please check it");
}
MAX_FILE_NUM = maxFileNum;
MAX_FILE_SIZE = maxFileSize;
}
public static Logger[] getLogs(DynamicLog dynLog) {
Logger logger = loggers.get(dynLog);
if (logger == null) {
synchronized (loggers) {
logger = loggers.get(dynLog);
if (logger == null) {
logger = initLogger(dynLog);
}
}
}
return new Logger[] {logger};
}
private static Logger initLogger(DynamicLog dynLog) {
if (printOnStd) {
Logger reallogger = Logger.getLogger(dynLog.getKey());
final ConsoleLogger logger = new ConsoleLogger(dynLog.getKey(), reallogger);
PatternLayout layout = new PatternLayout(LAYOUT_PATTERN);
ConsoleAppender appender = new ConsoleAppender(layout, ConsoleAppender.SYSTEM_OUT);
reallogger.removeAllAppenders();
reallogger.addAppender(appender);
reallogger.setLevel(level);
reallogger.setAdditivity(false);
loggers.putIfAbsent(dynLog, logger);
return logger;
} else {
Logger logger = makeLogger(dynLog.getKey(), dynLog.getDefaultLogFileName());
loggers.putIfAbsent(dynLog, logger);
return logger;
}
}
protected static Logger makeLogger(String loggerName, String filename) {
final Logger logger = Logger.getLogger(loggerName);
PatternLayout layout = new PatternLayout(LAYOUT_PATTERN);
File dir = new File(logsDir);
if (!dir.exists()) {
dir.mkdirs();
}
String logFileName = logsDir + "/" + filename;
if (logsSuffix != null) {
logFileName = logFileName.substring(0, logFileName.length() - 4) + "-" + logsSuffix
+ ".log";
}
System.out.println("new_log_path:" + logFileName);
RollingFileAppender appender;
try {
appender = new TimedFlushDailyRollingFileAppender(layout, logFileName);
appender.setAppend(true);
appender.setEncoding("UTF-8");
appender.setName(DAILY_APPENDER_NAME);
appender.setBufferSize(LOG_CACHE_SIZE);
appender.setBufferedIO(true);
appender.setImmediateFlush(false);
appender.setMaxBackupIndex(MAX_FILE_NUM);
appender.setMaxFileSize(MAX_FILE_SIZE);
appender.activateOptions();
} catch (IOException e) {
throw new RuntimeException(e);
}
logger.removeAllAppenders();
logger.addAppender(appender);
logger.setLevel(level);
logger.setAdditivity(false);
return logger;
}
}
java/common/src/main/java/org/ray/util/logger/LogLevel.java
-29
View File
@@ -1,29 +0,0 @@
package org.ray.util.logger;
public enum LogLevel {
ERROR("error", 0),
WARN("warn", 1),
INFO("info", 2),
DEBUG("debug", 3);
private final String name;
private final int index;
LogLevel(String name, int index) {
this.name = name;
this.index = index;
}
public static LogLevel of(String name) {
for (LogLevel level : values()) {
if (level.name.equals(name)) {
return level;
}
}
return null;
}
public Boolean needLog(LogLevel level) {
return level.index <= this.index;
}
}
java/common/src/main/java/org/ray/util/logger/RayLog.java
+32 -4
View File
@@ -1,17 +1,45 @@
package org.ray.util.logger;
import org.ray.util.SystemUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Dynamic loggers in Ray.
* loggers in Ray.
* 1. core logger is used for internal Ray status logging.
* 2. rapp for ray applications logging.
*/
public class RayLog {
/**
* for ray itself.
*/
public static final DynamicLog core = DynamicLog.registerName("core");
public static Logger core;
/**
* for ray's app's log.
* for ray app.
*/
public static DynamicLog rapp = core; //DynamicLog.registerName("rapp");
public static Logger rapp;
/**
* it must be called before using Ray loggers,
* or the dynamic update does not work.
* @param workingDir store the logs under params.working_directory
*/
public static void init(String workingDir) {
String loggingPath = System.getProperty("logging.path");
if (loggingPath == null) {
System.setProperty("logging.path", workingDir + "/logs");
}
String loggingFileName = System.getProperty("logging.file.name");
if (loggingFileName != null && loggingFileName.contains("*pid_suffix*")) {
loggingFileName = loggingFileName.replaceAll("\\*pid_suffix\\*",
String.valueOf(SystemUtil.pid()));
System.setProperty("logging.file.name", loggingFileName);
}
core = LoggerFactory.getLogger("core");
rapp = core;
}
}
java/common/src/main/java/org/ray/util/logger/TimedFlushDailyRollingFileAppender.java
-68
View File
@@ -1,68 +0,0 @@
package org.ray.util.logger;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import org.apache.log4j.Layout;
import org.apache.log4j.RollingFileAppender;
/**
* Normal log appender.
*/
public class TimedFlushDailyRollingFileAppender extends RollingFileAppender {
private static final Set<TimedFlushDailyRollingFileAppender> all = new HashSet<>();
static {
new TimedFlushLogThread().start();
}
public TimedFlushDailyRollingFileAppender() {
super();
synchronized (all) {
all.add(this);
}
}
public TimedFlushDailyRollingFileAppender(Layout layout, String filename) throws IOException {
super(layout, filename);
synchronized (all) {
all.add(this);
}
}
private void flush() {
try {
if (!checkEntryConditions()) {
return;
}
qw.flush();
} catch (Exception e) {
e.printStackTrace();
}
}
private static class TimedFlushLogThread extends Thread {
public TimedFlushLogThread() {
super();
setName("TimedFlushLogThread");
setDaemon(true);
}
public void run() {
while (true) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
synchronized (all) {
for (TimedFlushDailyRollingFileAppender appender : all) {
appender.flush();
}
}
}
}
}
}
java/common/src/main/resources/log4j.properties
+20
View File
@@ -0,0 +1,20 @@
# define default properties here
logging.level=INFO
logging.path=./run/logs
logging.file.name=core
logging.max.log.file.num=10
logging.max.log.file.size=500MB
log4j.rootLogger=${logging.level}, stdout, core
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %p %c{1} [%t]: %m%n
log4j.appender.core=org.apache.log4j.RollingFileAppender
log4j.appender.core.File=${logging.path}/${logging.file.name}.log
log4j.appender.core.Append=true
log4j.appender.core.MaxFileSize=${logging.max.log.file.size}
log4j.appender.core.MaxBackupIndex=${logging.max.log.file.num}
log4j.appender.core.layout=org.apache.log4j.PatternLayout
log4j.appender.core.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %p %c{1} [%t]: %m%n
java/pom.xml
+9 -6
View File
@@ -23,7 +23,8 @@
<properties>
<java.version>1.8</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.version>1.0</project.version>
<projetct.version>1.0</projetct.version>
<slf4j.version>1.7.25</slf4j.version>
</properties>
@@ -41,11 +42,6 @@
<version>2.47</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
<dependency>
<groupId>quartz</groupId>
<artifactId>quartz</artifactId>
@@ -127,6 +123,13 @@
<version>3.2.2</version>
</dependency>
<!--log-->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>${slf4j.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.github.davidmoten/flatbuffers-java -->
<dependency>
<groupId>com.github.davidmoten</groupId>
java/ray.config.ini
-10
View File
@@ -28,10 +28,6 @@ simple_fail_over = false
[ray.java]
;network_interface = en0
; set log level: debug info warn error
log_level = debug
;log_level = warn
[ray.java.start]
; run mode for this app SINGLE_PROCESS | SINGLE_BOX | CLUSTER
@@ -48,8 +44,6 @@ driver_id = 0123456789abcdef0123456789abcdef01234567
working_directory = %CONFIG_FILE_DIR%/run
logging_directory = %CONFIG_FILE_DIR%/run/logs
redis_port = 34111
num_local_schedulers = 1
@@ -66,10 +60,6 @@ object_store_occupied_memory_MB = 2
deploy = false
max_java_log_file_num = 10
max_java_log_file_size = 500MB
onebox_delay_seconds_before_run_app_logic = 0
; java class which main is served as the driver in a java worker
java/runtime-common/src/main/java/org/ray/core/RayRuntime.java
+4 -8
View File
@@ -1,6 +1,5 @@
package org.ray.core;
import java.io.Serializable;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
@@ -29,8 +28,6 @@ import org.ray.spi.PathConfig;
import org.ray.spi.RemoteFunctionManager;
import org.ray.util.config.ConfigReader;
import org.ray.util.exception.TaskExecutionException;
import org.ray.util.logger.DynamicLog;
import org.ray.util.logger.DynamicLogManager;
import org.ray.util.logger.RayLog;
/**
@@ -80,12 +77,11 @@ public abstract class RayRuntime implements RayApi {
}
}
configReader = new ConfigReader(configPath, updateConfigStr);
String loglevel = configReader.getStringValue("ray.java", "log_level", "debug",
"set the log output level(debug, info, warn, error)");
DynamicLog.setLogLevel(loglevel);
RayRuntime.params = new RayParameters(configReader);
DynamicLogManager.init(params.max_java_log_file_num, params.max_java_log_file_size);
RayLog.init(params.working_directory);
assert RayLog.core != null;
ins = instantiate(params);
assert (ins != null);
java/runtime-common/src/main/java/org/ray/core/model/RayParameters.java
-10
View File
@@ -49,9 +49,6 @@ public class RayParameters {
@AConfig(comment = "working directory")
public String working_directory = "./run";
@AConfig(comment = "directory for saving logs")
public String logging_directory = "./run/logs";
@AConfig(comment = "primary redis port")
public int redis_port = 34222;
@@ -109,16 +106,9 @@ public class RayParameters {
@AConfig(comment = "whether to use supreme failover strategy")
public boolean supremeFO = false;
@AConfig(comment = "the max num of java log of each java worker")
public int max_java_log_file_num = 10;
@AConfig(comment = "whether to disable process failover")
public boolean disable_process_failover = false;
@AConfig(comment = "the max size of each file of java worker log, could be set as 10KB, 10MB, "
+ "1GB or something similar")
public String max_java_log_file_size = "500MB";
@AConfig(comment = "delay seconds under onebox before app logic for debugging")
public int onebox_delay_seconds_before_run_app_logic = 0;
java/runtime-native/src/main/java/org/ray/runner/RunManager.java
+2 -3
View File
@@ -178,8 +178,6 @@ public class RunManager {
+ section + "node_ip_address=" + ip + ";"
+ section + "redis_address=" + redisAddr + ";"
+ section + "working_directory=" + workDir + ";"
+ section + "logging_directory=" + params.logging_directory + ";"
+ section + "working_directory=" + workDir + ";"
+ section + "run_mode=" + params.run_mode;
if (additionalConfigs.length() > 0) {
@@ -675,7 +673,8 @@ public class RunManager {
}
String jvmArgs = "";
jvmArgs += " -DlogOutput=" + params.logging_directory + "/workers/*pid_suffix*";
jvmArgs += " -Dlogging.path=" + params.working_directory + "/logs/workers";
jvmArgs += " -Dlogging.file.name=core-*pid_suffix*";
return buildJavaProcessCommand(
RunInfo.ProcessType.PT_WORKER,