Apache spark Spark自定义日志记录

我的IDE中有多个spark项目。默认情况下，spark正在spark/conf文件夹中拾取log4j.properties文件

因为我有多个spark项目，所以我希望有多个log4j.properties文件（每个项目一个）。可能作为项目代码的一部分（参考资料文件夹）

是否有一种方法可以获取指定的log4j.properies而不是默认的log4j.properties

注: 我试过这个

--driver-java-options "-Dlog4j.configuration=file:///usr/local/Cellar/apache-spark/2.4.1/libexec/conf/driver_log4j.properties"

它工作起来没有任何问题，但是我正在寻找下面这样的东西

package org.apache.spark.internal

import java.io.File
import java.text.SimpleDateFormat
import java.util.Date
import java.util.logging._

import scala.collection.mutable

protected [spark] object YLogger extends Serializable with Logging {

  private var ylogs_ = new mutable.HashMap[String, Logger]()

  private def initializeYLogging(className: String): Unit = {
    // Here we set log file onto user's home.
    val rootPath = System.getProperty("user.home")
    val logPath = rootPath + File.separator + className + ".log"
    logInfo(s"Create ylogger for class [${className}] with log file named [${logPath}]")
    val log_ = Logger.getLogger(className)
    val fileHandler = new FileHandler(logPath, false)
    val formatter = new Formatter {
      override def format(record: LogRecord): String = {
        val time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date)
        new StringBuilder()
          .append("[")
          .append(className)
          .append("]")
          .append("[")
          .append(time)
          .append("]")
          .append(":: ")
          .append(record.getMessage)
          .append("\r\n")
          .toString
      }
    }
    fileHandler.setFormatter(formatter)
    log_.addHandler(fileHandler)
    ylogs_.put(className, log_)
  }

  private def ylog(logName: String): Logger = {
    if (!ylogs_.contains(logName)) {
      initializeYLogging(logName)
    }
    ylogs_.get(logName).get
  }

  def ylogInfo(logName: String)(info: String): Unit = {
    if (ylog(logName).isLoggable(Level.INFO)) ylog(logName).info(info)
  }

  def ylogWarning(logName: String)(warning: String): Unit = {
    if (ylog(logName).isLoggable(Level.WARNING)) ylog(logName).warning(warning)
  }
}

YLogger.ylogInfo("logFileName") ("This is a log.")

但是，我想在创建spark logger时加载资源文件夹中的log4j.properties文件

class SparkLogger():
    def __init__(self, app_name, sparksession = None):
        self._spark = sparksession
        self.log4jLogger = None

        if self._spark is not None:
            sparkContext =self._spark.sparkContext
            self.log4jLogger = sparkContext._jvm.org.apache.log4j
            self.log4jLogger = self.log4jLogger.LogManager.getLogger(app_name)

    def info(self, info):
        if self.log4jLogger:
            self.log4jLogger.info(str(info))

    def error(self, info):
        if self.log4jLogger:
            self.log4jLogger.error(str(info))

    def warn(self, info):
        if self.log4jLogger:
            self.log4jLogger.warn(str(info))

    def debug(self, info):
        if self.log4jLogger:
            self.log4jLogger.debug(str(info))

---

我试图建立我的自定义日志，就像你在问题中描述的那样，但最终失败了。我得说那完全是浪费

最后，我选择了java.util.logging而不是log4j。实际上，它是JDK中的一个原始日志util。我使用它的目的是只想将自己的信息记录到指定的文件中

因此，课程如下所示

package org.apache.spark.internal

import java.io.File
import java.text.SimpleDateFormat
import java.util.Date
import java.util.logging._

import scala.collection.mutable

protected [spark] object YLogger extends Serializable with Logging {

  private var ylogs_ = new mutable.HashMap[String, Logger]()

  private def initializeYLogging(className: String): Unit = {
    // Here we set log file onto user's home.
    val rootPath = System.getProperty("user.home")
    val logPath = rootPath + File.separator + className + ".log"
    logInfo(s"Create ylogger for class [${className}] with log file named [${logPath}]")
    val log_ = Logger.getLogger(className)
    val fileHandler = new FileHandler(logPath, false)
    val formatter = new Formatter {
      override def format(record: LogRecord): String = {
        val time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date)
        new StringBuilder()
          .append("[")
          .append(className)
          .append("]")
          .append("[")
          .append(time)
          .append("]")
          .append(":: ")
          .append(record.getMessage)
          .append("\r\n")
          .toString
      }
    }
    fileHandler.setFormatter(formatter)
    log_.addHandler(fileHandler)
    ylogs_.put(className, log_)
  }

  private def ylog(logName: String): Logger = {
    if (!ylogs_.contains(logName)) {
      initializeYLogging(logName)
    }
    ylogs_.get(logName).get
  }

  def ylogInfo(logName: String)(info: String): Unit = {
    if (ylog(logName).isLoggable(Level.INFO)) ylog(logName).info(info)
  }

  def ylogWarning(logName: String)(warning: String): Unit = {
    if (ylog(logName).isLoggable(Level.WARNING)) ylog(logName).warning(warning)
  }
}

YLogger.ylogInfo("logFileName") ("This is a log.")

你可以像下面那样使用它

package org.apache.spark.internal

import java.io.File
import java.text.SimpleDateFormat
import java.util.Date
import java.util.logging._

import scala.collection.mutable

protected [spark] object YLogger extends Serializable with Logging {

  private var ylogs_ = new mutable.HashMap[String, Logger]()

  private def initializeYLogging(className: String): Unit = {
    // Here we set log file onto user's home.
    val rootPath = System.getProperty("user.home")
    val logPath = rootPath + File.separator + className + ".log"
    logInfo(s"Create ylogger for class [${className}] with log file named [${logPath}]")
    val log_ = Logger.getLogger(className)
    val fileHandler = new FileHandler(logPath, false)
    val formatter = new Formatter {
      override def format(record: LogRecord): String = {
        val time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date)
        new StringBuilder()
          .append("[")
          .append(className)
          .append("]")
          .append("[")
          .append(time)
          .append("]")
          .append(":: ")
          .append(record.getMessage)
          .append("\r\n")
          .toString
      }
    }
    fileHandler.setFormatter(formatter)
    log_.addHandler(fileHandler)
    ylogs_.put(className, log_)
  }

  private def ylog(logName: String): Logger = {
    if (!ylogs_.contains(logName)) {
      initializeYLogging(logName)
    }
    ylogs_.get(logName).get
  }

  def ylogInfo(logName: String)(info: String): Unit = {
    if (ylog(logName).isLoggable(Level.INFO)) ylog(logName).info(info)
  }

  def ylogWarning(logName: String)(warning: String): Unit = {
    if (ylog(logName).isLoggable(Level.WARNING)) ylog(logName).warning(warning)
  }
}

YLogger.ylogInfo("logFileName") ("This is a log.")

---

它的使用非常简单，我希望我的回答能对你有所帮助。

我尝试像你在问题中描述的那样构建自定义日志，但最终失败了。我得说那完全是浪费

最后，我选择了java.util.logging而不是log4j。实际上，它是JDK中的一个原始日志util。我使用它的目的是只想将自己的信息记录到指定的文件中

因此，课程如下所示

package org.apache.spark.internal

import java.io.File
import java.text.SimpleDateFormat
import java.util.Date
import java.util.logging._

import scala.collection.mutable

protected [spark] object YLogger extends Serializable with Logging {

  private var ylogs_ = new mutable.HashMap[String, Logger]()

  private def initializeYLogging(className: String): Unit = {
    // Here we set log file onto user's home.
    val rootPath = System.getProperty("user.home")
    val logPath = rootPath + File.separator + className + ".log"
    logInfo(s"Create ylogger for class [${className}] with log file named [${logPath}]")
    val log_ = Logger.getLogger(className)
    val fileHandler = new FileHandler(logPath, false)
    val formatter = new Formatter {
      override def format(record: LogRecord): String = {
        val time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date)
        new StringBuilder()
          .append("[")
          .append(className)
          .append("]")
          .append("[")
          .append(time)
          .append("]")
          .append(":: ")
          .append(record.getMessage)
          .append("\r\n")
          .toString
      }
    }
    fileHandler.setFormatter(formatter)
    log_.addHandler(fileHandler)
    ylogs_.put(className, log_)
  }

  private def ylog(logName: String): Logger = {
    if (!ylogs_.contains(logName)) {
      initializeYLogging(logName)
    }
    ylogs_.get(logName).get
  }

  def ylogInfo(logName: String)(info: String): Unit = {
    if (ylog(logName).isLoggable(Level.INFO)) ylog(logName).info(info)
  }

  def ylogWarning(logName: String)(warning: String): Unit = {
    if (ylog(logName).isLoggable(Level.WARNING)) ylog(logName).warning(warning)
  }
}

YLogger.ylogInfo("logFileName") ("This is a log.")

你可以像下面那样使用它

package org.apache.spark.internal

import java.io.File
import java.text.SimpleDateFormat
import java.util.Date
import java.util.logging._

import scala.collection.mutable

protected [spark] object YLogger extends Serializable with Logging {

  private var ylogs_ = new mutable.HashMap[String, Logger]()

  private def initializeYLogging(className: String): Unit = {
    // Here we set log file onto user's home.
    val rootPath = System.getProperty("user.home")
    val logPath = rootPath + File.separator + className + ".log"
    logInfo(s"Create ylogger for class [${className}] with log file named [${logPath}]")
    val log_ = Logger.getLogger(className)
    val fileHandler = new FileHandler(logPath, false)
    val formatter = new Formatter {
      override def format(record: LogRecord): String = {
        val time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date)
        new StringBuilder()
          .append("[")
          .append(className)
          .append("]")
          .append("[")
          .append(time)
          .append("]")
          .append(":: ")
          .append(record.getMessage)
          .append("\r\n")
          .toString
      }
    }
    fileHandler.setFormatter(formatter)
    log_.addHandler(fileHandler)
    ylogs_.put(className, log_)
  }

  private def ylog(logName: String): Logger = {
    if (!ylogs_.contains(logName)) {
      initializeYLogging(logName)
    }
    ylogs_.get(logName).get
  }

  def ylogInfo(logName: String)(info: String): Unit = {
    if (ylog(logName).isLoggable(Level.INFO)) ylog(logName).info(info)
  }

  def ylogWarning(logName: String)(warning: String): Unit = {
    if (ylog(logName).isLoggable(Level.WARNING)) ylog(logName).warning(warning)
  }
}

YLogger.ylogInfo("logFileName") ("This is a log.")

---

它的使用非常简单，我希望我的答案能对你有所帮助。

你必须在

log4j文件中定义
application\u name log
logger属性。当您使用
applicationon\u name
调用get logger方法时，您将能够访问定制的应用程序基础日志生成
 您必须在
log4j文件中定义
application\u name log
记录器属性。当您使用
applicationon\u name
调用get logger方法时，您将能够访问定制的应用程序基础日志生成