update to slf4j, remove DynamicLog (#2384)

java/common/pom.xml

@@ -20,8 +20,8 @@
 
     <dependencies>
         <dependency>
-            <groupId>log4j</groupId>
-            <artifactId>log4j</artifactId>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-log4j12</artifactId>
         </dependency>
         <dependency>
             <groupId>quartz</groupId>

java/common/src/main/java/org/ray/util/logger/ConsoleLogger.java

@@ -1,46 +0,0 @@
-package org.ray.util.logger;
-
-import org.apache.log4j.Logger;
-
-/**
- * A logger which prints output to console.
- */
-public class ConsoleLogger extends Logger {
-
-  final Logger realLogger;
-
-  protected ConsoleLogger(String name, Logger realLogger) {
-    super(name);
-    this.realLogger = realLogger;
-  }
-
-  @Override
-  public void debug(Object log) {
-    realLogger.debug("(" + this.getName() + ") " + log);
-  }
-
-  @Override
-  public void error(Object log) {
-    realLogger.error("(" + this.getName() + ") " + log);
-  }
-
-  @Override
-  public void error(Object log, Throwable e) {
-    realLogger.error("(" + this.getName() + ") " + log, e);
-  }
-
-  @Override
-  public void info(Object log) {
-    realLogger.info("(" + this.getName() + ") " + log);
-  }
-
-  @Override
-  public void warn(Object log) {
-    realLogger.warn("(" + this.getName() + ") " + log);
-  }
-
-  @Override
-  public void warn(Object log, Throwable e) {
-    realLogger.warn("(" + this.getName() + ") " + log, e);
-  }
-}

java/common/src/main/java/org/ray/util/logger/DynamicLog.java

@@ -1,258 +0,0 @@
-package org.ray.util.logger;
-
-import java.util.Collection;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import org.apache.log4j.Logger;
-import org.ray.util.CommonUtil;
-
-/**
- * Dynamic logger without properties configuration file.
- */
-public class DynamicLog {
-
-  static final ThreadLocal<String> PREFIX = new ThreadLocal<>();
-
-  private static LogLevel logLevel = LogLevel.DEBUG;
-
-  private static Boolean logLevelSetFlag = false;
-  private static Map<String/*Samplekey*/, SampleStatis> sampleStatis = new ConcurrentHashMap<>();
-  private final String key;
-
-  private DynamicLog(String key) {
-    this.key = key;
-  }
-
-  public static String getContextPrefix() {
-    return PREFIX.get();
-  }
-
-  /**
-   * set the context prefix for all logs.
-   */
-  public static void setContextPrefix(String prefix) {
-    PREFIX.set(prefix);
-  }
-
-  /**
-   * set the level for all logs.
-   */
-  public static void setLogLevel(String level) {
-    if (logLevelSetFlag) { /* one shot, avoid the risk of multithreading */
-      return;
-    }
-    logLevelSetFlag = true;
-    logLevel = LogLevel.of(level);
-  }
-
-  public static DynamicLog registerName(String name) {
-    return DynamicLogNameRegister.registerName(name);
-  }
-
-  public static Collection<DynamicLog> values() {
-    return DynamicLogNameRegister.names.values();
-  }
-
-  @Override
-  public int hashCode() {
-    return this.toString().hashCode();
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    return this.toString().equals(o.toString());
-  }
-
-  @Override
-  public String toString() {
-    return this.getKey();
-  }
-
-  public String getKey() {
-    return this.key;
-  }
-
-  public void debug(String log) {
-    if (!getenumLogLevel().needLog(LogLevel.DEBUG)) {
-      return;
-    }
-    log = wrap("debug", log);
-    Logger[] loggers = DynamicLogManager.getLogs(this);
-    for (Logger logger : loggers) {
-      logger.debug(log);
-    }
-  }
-
-  private static LogLevel getenumLogLevel() {
-    return logLevel;
-  }
-
-  private String wrap(String level, String log) {
-    StackTraceElement[] stes = Thread.currentThread().getStackTrace();
-    String ret = "[" + level + "]" + "[" + stes[3].getFileName() + ":" + stes[3].getLineNumber()
-        + "] - " + (log == null ? "" : log);
-    String prefix = PREFIX.get();
-    if (prefix != null) {
-      ret = "[" + prefix + "]" + ret;
-    }
-    return ret;
-  }
-
-  public void info(String log) {
-    if (!getenumLogLevel().needLog(LogLevel.INFO)) {
-      return;
-    }
-    log = wrap("info", log);
-    Logger[] loggers = DynamicLogManager.getLogs(this);
-    for (Logger logger : loggers) {
-      logger.info(log);
-    }
-  }
-
-  public void warn(String log) {
-    if (!getenumLogLevel().needLog(LogLevel.WARN)) {
-      return;
-    }
-    log = wrap("warn", log);
-    Logger[] loggers = DynamicLogManager.getLogs(this);
-    for (Logger logger : loggers) {
-      logger.warn(log);
-    }
-  }
-
-  public void warn(String log, Throwable e) {
-    if (!getenumLogLevel().needLog(LogLevel.WARN)) {
-      return;
-    }
-    log = wrap("warn", log);
-    Logger[] loggers = DynamicLogManager.getLogs(this);
-    for (Logger logger : loggers) {
-      logger.warn(log, e);
-    }
-  }
-
-  public void error(String log, Throwable e) {
-    if (!getenumLogLevel().needLog(LogLevel.ERROR)) {
-      return;
-    }
-    log = wrap("error", log);
-    if (e == null) {
-      error(log);
-      return;
-    }
-
-    Logger[] loggers = DynamicLogManager.getLogs(this);
-    for (Logger logger : loggers) {
-      logger.error(log, e);
-    }
-  }
-
-  public void error(String log) {
-    if (!getenumLogLevel().needLog(LogLevel.ERROR)) {
-      return;
-    }
-    log = wrap("error", log);
-    Logger[] loggers = DynamicLogManager.getLogs(this);
-    for (Logger logger : loggers) {
-      logger.error(log);
-    }
-  }
-
-  public void error(Throwable e) {
-    if (!getenumLogLevel().needLog(LogLevel.ERROR)) {
-      return;
-    }
-    String log = wrap("error", e == null ? null : e.getMessage());
-    if (e == null) {
-      error(log);
-      return;
-    }
-    Logger[] loggers = DynamicLogManager.getLogs(this);
-    for (Logger logger : loggers) {
-      logger.error(log, e);
-    }
-  }
-
-  /**
-   * Print sample error log.
-   */
-  public boolean sampleError(Object sampleKeyO, String log, Throwable e) {
-    String sampleKey = sampleKeyO.toString();
-    try {
-      SampleStatis ss = sampleStatis.computeIfAbsent(sampleKey, k -> new SampleStatis());
-      if (ss.gamble()) {
-        Logger[] loggers = DynamicLogManager.getLogs(this);
-        for (Logger logger : loggers) {
-          if (e != null) {
-            logger.error("[" + sampleKey + "] - " + log, e);
-          } else {
-            logger.error("[" + sampleKey + "] - " + log);
-          }
-        }
-        return true;
-      } else {
-        return false;
-      }
-    } finally {
-      if (sampleStatis.size() > 100000) {
-        sampleStatis = new ConcurrentHashMap<>();
-      }
-    }
-  }
-
-  public String getDefaultLogFileName() {
-    return this.key + ".log";
-  }
-
-  //statistic for sampling
-  private static class SampleStatis {
-
-    int total;
-
-    public boolean gamble() {
-      int randomRange;
-
-      if (total < 100) {
-        randomRange = 1;
-      } else if (total < 1000) {
-        randomRange = 1000;
-      } else if (total < 100000) {
-        randomRange = 10000;
-      } else if (total < 1000000) {
-        randomRange = 100000;
-      } else {
-        total = 0;
-        randomRange = 1;
-      }
-      if (CommonUtil.getRandom(randomRange) == 0) {
-        total++;
-        return true;
-      } else {
-        total++;
-        return false;
-      }
-    }
-  }
-
-  public static class DynamicLogNameRegister {
-
-    static final Map<String, DynamicLog> names = new ConcurrentHashMap<>();
-
-    public static DynamicLog registerName(String name) {
-      DynamicLog ret = names.get(name);
-      if (ret != null) {
-        return ret;
-      }
-      synchronized (names) {
-        ret = names.get(name);
-        if (ret != null) {
-          return ret;
-        }
-        ret = new DynamicLog(name);
-        names.put(name, ret);
-        return ret;
-      }
-    }
-  }
-
-}

java/common/src/main/java/org/ray/util/logger/DynamicLogManager.java

@@ -1,160 +0,0 @@
-package org.ray.util.logger;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.concurrent.ConcurrentHashMap;
-import org.apache.log4j.ConsoleAppender;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.log4j.PatternLayout;
-import org.apache.log4j.RollingFileAppender;
-import org.ray.util.SystemUtil;
-
-/**
- * Manager for dynamic loggers.
- */
-public class DynamicLogManager {
-
-  protected static final String DAY_DATE_PATTERN = "'.'yyyy-MM-dd";
-  private static final int LOG_CACHE_SIZE = 32 * 1024;
-  //    private final static String                          HOUR_DATE_PATTERN     = "'
-  // .'yyyy-MM-dd_HH";
-  //    private final static String                          GBK                   = "GBK";
-  private static final String DAILY_APPENDER_NAME = "_DAILY_APPENDER_NAME";
-  //    private final static String                          CONSOLE_APPENDER_NAME =
-  // "_CONSOLE_APPENDER_NAME";
-  private static final String LAYOUT_PATTERN = "%d [%t]%m%n";
-  private static final ConcurrentHashMap<DynamicLog, Logger> loggers = new ConcurrentHashMap<>();
-  //whether to print the log on std(ie. console)
-  public static boolean printOnStd = false;
-  //the root directory of log files
-  public static String logsDir;
-  public static String logsSuffix;
-  public static Level level = Level.DEBUG; //Level.INFO;
-  private static int MAX_FILE_NUM = 10;
-
-  private static String MAX_FILE_SIZE = "500MB";
-
-  private static boolean initFinished = false;
-
-
-  /**
-   * init from system properties
-   * -DlogOutput=console/file_path
-   * if file_path contains *pid*, it will be replaced with real PID of this JAVA process
-   * if file_path contains *pid_suffix*, all log file will append the suffix -> xxx-pid.log
-   */
-  static {
-    String logOutput = System.getProperty("logOutput");
-    if (null == logOutput
-        || logOutput.equalsIgnoreCase("console")
-        || logOutput.equalsIgnoreCase("std")
-        || logOutput.equals("")) {
-      DynamicLogManager.printOnStd = true;
-      System.out.println("config log output as std");
-    } else {
-      if (logOutput.contains("*pid*")) {
-        logOutput = logOutput.replaceAll("\\*pid\\*", String.valueOf(SystemUtil.pid()));
-      }
-      if (logOutput.contains("*pid_suffix*")) {
-        logOutput = logOutput.replaceAll("\\*pid_suffix\\*", "");
-        if (logOutput.endsWith("/")) {
-          logOutput = logOutput.substring(0, logOutput.length() - 1);
-        }
-        DynamicLogManager.logsSuffix = String.valueOf(SystemUtil.pid());
-      }
-      System.out.println("config log output as " + logOutput);
-      DynamicLogManager.logsDir = logOutput;
-    }
-    String logLevel = System.getProperty("logLevel");
-    if (logLevel != null && logLevel.equals("debug")) {
-      level = Level.DEBUG;
-    }
-  }
-
-  public static synchronized void init(int maxFileNum, String maxFileSize) {
-    if (initFinished) {
-      return;
-    }
-    initFinished = true;
-    System.out.println(
-        "DynamicLogManager init with maxFileNum:" + maxFileNum + " maxFileSize:" + maxFileSize);
-    if (loggers.size() > 0) {
-      System.err
-          .println("already have logger be maked before init log file system, please check it");
-    }
-    MAX_FILE_NUM = maxFileNum;
-    MAX_FILE_SIZE = maxFileSize;
-  }
-
-  public static Logger[] getLogs(DynamicLog dynLog) {
-    Logger logger = loggers.get(dynLog);
-    if (logger == null) {
-      synchronized (loggers) {
-        logger = loggers.get(dynLog);
-        if (logger == null) {
-          logger = initLogger(dynLog);
-        }
-      }
-    }
-    return new Logger[] {logger};
-  }
-
-  private static Logger initLogger(DynamicLog dynLog) {
-    if (printOnStd) {
-      Logger reallogger = Logger.getLogger(dynLog.getKey());
-      final ConsoleLogger logger = new ConsoleLogger(dynLog.getKey(), reallogger);
-      PatternLayout layout = new PatternLayout(LAYOUT_PATTERN);
-      ConsoleAppender appender = new ConsoleAppender(layout, ConsoleAppender.SYSTEM_OUT);
-      reallogger.removeAllAppenders();
-      reallogger.addAppender(appender);
-      reallogger.setLevel(level);
-      reallogger.setAdditivity(false);
-      loggers.putIfAbsent(dynLog, logger);
-      return logger;
-    } else {
-      Logger logger = makeLogger(dynLog.getKey(), dynLog.getDefaultLogFileName());
-      loggers.putIfAbsent(dynLog, logger);
-      return logger;
-    }
-
-  }
-
-  protected static Logger makeLogger(String loggerName, String filename) {
-    final Logger logger = Logger.getLogger(loggerName);
-    PatternLayout layout = new PatternLayout(LAYOUT_PATTERN);
-    File dir = new File(logsDir);
-    if (!dir.exists()) {
-      dir.mkdirs();
-    }
-    String logFileName = logsDir + "/" + filename;
-    if (logsSuffix != null) {
-      logFileName = logFileName.substring(0, logFileName.length() - 4) + "-" + logsSuffix
-          + ".log";
-    }
-    System.out.println("new_log_path:" + logFileName);
-    RollingFileAppender appender;
-    try {
-      appender = new TimedFlushDailyRollingFileAppender(layout, logFileName);
-      appender.setAppend(true);
-      appender.setEncoding("UTF-8");
-      appender.setName(DAILY_APPENDER_NAME);
-      appender.setBufferSize(LOG_CACHE_SIZE);
-      appender.setBufferedIO(true);
-      appender.setImmediateFlush(false);
-      appender.setMaxBackupIndex(MAX_FILE_NUM);
-      appender.setMaxFileSize(MAX_FILE_SIZE);
-      appender.activateOptions();
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
-
-    logger.removeAllAppenders();
-    logger.addAppender(appender);
-
-    logger.setLevel(level);
-    logger.setAdditivity(false);
-    return logger;
-  }
-
-}

java/common/src/main/java/org/ray/util/logger/LogLevel.java

@@ -1,29 +0,0 @@
-package org.ray.util.logger;
-
-public enum LogLevel {
-  ERROR("error", 0),
-  WARN("warn", 1),
-  INFO("info", 2),
-  DEBUG("debug", 3);
-
-  private final String name;
-  private final int index;
-
-  LogLevel(String name, int index) {
-    this.name = name;
-    this.index = index;
-  }
-
-  public static LogLevel of(String name) {
-    for (LogLevel level : values()) {
-      if (level.name.equals(name)) {
-        return level;
-      }
-    }
-    return null;
-  }
-
-  public Boolean needLog(LogLevel level) {
-    return level.index <= this.index;
-  }
-}

java/common/src/main/java/org/ray/util/logger/RayLog.java

@@ -1,17 +1,45 @@
 package org.ray.util.logger;
 
+import org.ray.util.SystemUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
- * Dynamic loggers in Ray.
+ * loggers in Ray.
+ *   1. core logger is used for internal Ray status logging.
+ *   2. rapp for ray applications logging.
  */
 public class RayLog {
 
   /**
    * for ray itself.
    */
-  public static final DynamicLog core = DynamicLog.registerName("core");
+  public static Logger core;
 
   /**
-   * for ray's app's log.
+   * for ray app.
    */
-  public static DynamicLog rapp = core; //DynamicLog.registerName("rapp");
+  public static Logger rapp;
+
+  /**
+   * it must be called before using Ray loggers,
+   *  or the dynamic update does not work.
+   * @param workingDir store the logs under params.working_directory
+   */
+  public static void init(String workingDir) {
+    String loggingPath = System.getProperty("logging.path");
+    if (loggingPath == null) {
+      System.setProperty("logging.path", workingDir + "/logs");
+    }
+    String loggingFileName = System.getProperty("logging.file.name");
+    if (loggingFileName != null && loggingFileName.contains("*pid_suffix*")) {
+      loggingFileName = loggingFileName.replaceAll("\\*pid_suffix\\*",
+              String.valueOf(SystemUtil.pid()));
+      System.setProperty("logging.file.name", loggingFileName);
+    }
+
+    core = LoggerFactory.getLogger("core");
+
+    rapp = core;
+  }
 }

java/common/src/main/java/org/ray/util/logger/TimedFlushDailyRollingFileAppender.java

@@ -1,68 +0,0 @@
-package org.ray.util.logger;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
-import org.apache.log4j.Layout;
-import org.apache.log4j.RollingFileAppender;
-
-/**
- * Normal log appender.
- */
-public class TimedFlushDailyRollingFileAppender extends RollingFileAppender {
-
-  private static final Set<TimedFlushDailyRollingFileAppender> all = new HashSet<>();
-
-  static {
-    new TimedFlushLogThread().start();
-  }
-
-  public TimedFlushDailyRollingFileAppender() {
-    super();
-    synchronized (all) {
-      all.add(this);
-    }
-  }
-
-  public TimedFlushDailyRollingFileAppender(Layout layout, String filename) throws IOException {
-    super(layout, filename);
-    synchronized (all) {
-      all.add(this);
-    }
-  }
-
-  private void flush() {
-    try {
-      if (!checkEntryConditions()) {
-        return;
-      }
-      qw.flush();
-    } catch (Exception e) {
-      e.printStackTrace();
-    }
-  }
-
-  private static class TimedFlushLogThread extends Thread {
-
-    public TimedFlushLogThread() {
-      super();
-      setName("TimedFlushLogThread");
-      setDaemon(true);
-    }
-
-    public void run() {
-      while (true) {
-        try {
-          Thread.sleep(1000);
-        } catch (InterruptedException e) {
-          e.printStackTrace();
-        }
-        synchronized (all) {
-          for (TimedFlushDailyRollingFileAppender appender : all) {
-            appender.flush();
-          }
-        }
-      }
-    }
-  }
-}

java/common/src/main/resources/log4j.properties

@@ -0,0 +1,20 @@
+# define default properties here
+logging.level=INFO
+logging.path=./run/logs
+logging.file.name=core
+logging.max.log.file.num=10
+logging.max.log.file.size=500MB
+
+log4j.rootLogger=${logging.level}, stdout, core
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %p %c{1} [%t]: %m%n
+
+log4j.appender.core=org.apache.log4j.RollingFileAppender
+log4j.appender.core.File=${logging.path}/${logging.file.name}.log
+log4j.appender.core.Append=true
+log4j.appender.core.MaxFileSize=${logging.max.log.file.size}
+log4j.appender.core.MaxBackupIndex=${logging.max.log.file.num}
+log4j.appender.core.layout=org.apache.log4j.PatternLayout
+log4j.appender.core.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %p %c{1} [%t]: %m%n