应用系统日志采集解决方案

1. 采集范围
   每一次用户请求的请求信息。
2. 数据量大
3. 尽量减少现有系统的改动

数据流图

2.2  文件内容

1. pom.xml

   <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
       xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
       <modelVersion>4.0.0</modelVersion>
       <groupId>cn.sinobest.asj</groupId>
       <artifactId>log-service</artifactId>
       <packaging>war</packaging>
       <version>0.0.1-SNAPSHOT</version>
       <name>log-service Maven Webapp</name>
       <url>http://maven.apache.org</url>
       <dependencies>
           <dependency>
               <groupId>junit</groupId>
               <artifactId>junit</artifactId>
               <version>3.8.1</version>
               <scope>test</scope>
           </dependency>
           <dependency>
               <groupId>log4j</groupId>
               <artifactId>log4j</artifactId>
               <version>1.2.16</version>
           </dependency>
           <dependency>
               <groupId>commons-logging</groupId>
               <artifactId>commons-logging</artifactId>
               <version>1.1.1</version>
           </dependency>
           <!-- for log to Flume -->
           <dependency>
               <groupId>org.apache.flume.flume-ng-clients</groupId>
               <artifactId>flume-ng-log4jappender</artifactId>
               <version>1.6.0</version>
           </dependency>
           <!-- for jax-ws -->
           <dependency>
               <groupId>com.sun.xml.ws</groupId>
               <artifactId>jaxws-rt</artifactId>
               <version>2.2.10</version>
           </dependency>
           <!-- for test the log content is a json-format or not -->
           <dependency>
               <groupId>org.mongodb</groupId>
               <artifactId>mongo-java-driver</artifactId>
               <version>2.13.0</version>
           </dependency>
       </dependencies>
       <build>
           <finalName>log-service</finalName>
       </build>
   </project>

2. web.xml

   <?xml version="1.0" encoding="UTF-8"?>
   <web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
            xmlns="http://java.sun.com/xml/ns/javaee"
            xmlns:web="http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd"
            xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd"
            id="WebApp_ID" version="3.0" metadata-complete="false">
       <display-name>Archetype Created Web Application</display-name>
   </web-app>

   注意：如果是servlet3.0以下的版本，需要额外的配置。

3. log4j.properties

   # 配置Log4jAppender，能写日志到Flume
   log4j.appender.flumeAvro=org.apache.flume.clients.log4jappender.Log4jAppender
   log4j.appender.flumeAvro.Hostname=localhost
   log4j.appender.flumeAvro.Port=44444
   log4j.appender.flumeAvro.UnsafeMode=true
   log4j.appender.flumeAvro.layout=org.apache.log4j.PatternLayout
   log4j.appender.flumeAvro.layout.ConversionPattern=%m
   # set root logger
   log4j.rootLogger=INFO, flumeAvro

4. ISALog.java

   package cn.sinobest.asj.log;
   import javax.jws.WebParam;
   import javax.jws.WebService;
   import cn.sinobest.asj.log.exception.InvalidFormatExceptioin;
   import cn.sinobest.asj.log.exception.InvalidGradeException;
   /**
    * SINOBEST ASJ Log - 为实现日志的统一采集和管理.
    * 
    * @author lijinlong
    * 
    */
   @WebService
   public interface ISALog {
       /**
        * 日志记录.
        * 
        * @param grade
        *            日志等级描述 - 忽略大小写.
        * @param content
        *            日志内容 - 需要为JSON格式的字符串.
        */
       public void log(@WebParam(name = "grade") String grade,
               @WebParam(name = "content") String content)
               throws InvalidGradeException, InvalidFormatExceptioin;
   }

5. SALogImpl.java

   package cn.sinobest.asj.log;
   import javax.jws.WebService;
   import org.apache.commons.logging.Log;
   import org.apache.commons.logging.LogFactory;
   import cn.sinobest.asj.log.exception.InvalidFormatExceptioin;
   import cn.sinobest.asj.log.exception.InvalidGradeException;
   import cn.sinobest.asj.log.util.MessageTemplate;
   import cn.sinobest.asj.log.util.ValidGrade;
   import com.mongodb.util.JSON;
   @WebService(endpointInterface = "cn.sinobest.asj.log.ISALog")
   public class SALogImpl implements ISALog {
       static final Log log = LogFactory.getLog(SALogImpl.class);
       public void log(String grade, String content) throws InvalidGradeException,
               InvalidFormatExceptioin {
           checkGrade(grade);
           checkContent(content);
           ValidGrade vg = ValidGrade.valueOf(grade.toUpperCase());
           log(vg, content);
       }
       /**
        * 根据日志等级，调用{@link log}的不同方法记录日志.
        * 
        * @param vg
        *            日志等级
        * @param content
        *            日志内容
        */
       private void log(ValidGrade vg, String content) {
           switch (vg) {
           case DEBUG:
               log.debug(content);
               break;
           case INFO:
               log.info(content);
               break;
           case WARN:
               log.warn(content);
               break;
           case ERROR:
               log.error(content);
               break;
           default:
               break;
           }
       }
       /**
        * 检查日志等级的有效性.
        * 
        * @param grade
        *            日志等级描述.
        * @throws InvalidGradeException
        *             当日志等级无效时，抛出此异常.
        */
       private void checkGrade(String grade) throws InvalidGradeException {
           boolean valid = ValidGrade.isValid(grade);
           if (!valid) {
               String message = String.format(MessageTemplate.INVALID_GRADE,
                       grade, ValidGrade.getEnumContent());
               throw new InvalidGradeException(message);
           }
       }
       /**
        * 检查日志内容格式的有效性.<br>
        * 要求为JSON格式的字符串.
        * 
        * @param content
        *            日志内容.
        * @throws InvalidFormatExceptioin
        *             当日志内容格式无效时，抛出此异常.
        */
       private void checkContent(String content) throws InvalidFormatExceptioin {
           boolean valid = true;
           if (content == null || content.isEmpty()) {
               valid = false;
           } else {
               try {
                   JSON.parse(content);
                   valid = true;
               } catch (com.mongodb.util.JSONParseException e) {
                   valid = false;
               }
           }
           if (!valid) {
               String message = String.format(MessageTemplate.INVALID_FORMAT,
                       content);
               throw new InvalidFormatExceptioin(message);
           }
       }
       /**
        * just for test.
        * 
        * @param args
        */
       public static void main(String[] args) {
           String[][] data = { { "info", "{‘name‘:‘ljl‘,‘age‘:26}" },
                   { "INFO", "trouble is a friend." },
                   { "JOKE", "{‘message‘:‘I am feeling down.‘}" } };
           ISALog ilog = new SALogImpl();
           for (String[] dat : data) {
               String grade = dat[0];
               String content = dat[1];
               try {
                   ilog.log(grade, content);
               } catch (Exception e) {
                   e.printStackTrace();
               }
           }
       }
   }

6. InvalidGradeException.java

   package cn.sinobest.asj.log.exception;
   /**
    * 表示无效的日志等级.
    * @author lijinlong
    *
    */
   public class InvalidGradeException extends Exception {
       private static final long serialVersionUID = 1341726127995938030L;
       public InvalidGradeException(String message) {
           super(message);
       }
   }

7. InvalidFormatExceptioin.java

   package cn.sinobest.asj.log.exception;
   /**
    * 表示无效的日志等级.
    * @author lijinlong
    *
    */
   public class InvalidGradeException extends Exception {
       private static final long serialVersionUID = 1341726127995938030L;
       public InvalidGradeException(String message) {
           super(message);
       }
   }

8. ValidGrade.java

   package cn.sinobest.asj.log.util;
   /**
    * 有效的日志等级.
    * 
    * @author lijinlong
    * 
    */
   public enum ValidGrade {
       DEBUG, INFO, WARN, ERROR;
       /** 有效日志等级的枚举内容 */
       private static String enumContent;
       /**
        * 获取所有有效的日志等级.
        * 
        * @return
        */
       public static String getEnumContent() {
           if (enumContent != null && !enumContent.isEmpty())
               return enumContent;
           ValidGrade[] vgs = ValidGrade.values();
           StringBuilder builder = new StringBuilder(30);
           for (ValidGrade vg : vgs) {
               builder.append(vg).append(",");
           }
           builder.delete(builder.length() - 1, builder.length());
           enumContent = builder.toString();
           return enumContent;
       }
       
       /**
        * 判断日志等级是否有效.
        * @param grade 日志等级 - 忽略大小写.
        * @return
        */
       public static boolean isValid(String grade) {
           if (grade == null || grade.isEmpty())
               return false;
           
           boolean result = false;
           
           final String GRADE = grade.toUpperCase();
           ValidGrade[] vgs = ValidGrade.values();
           for (ValidGrade vg : vgs) {
               if (vg.toString().equals(GRADE)) {
                   result = true;
                   break;
               }
           }
           
           return result;
       }
       
       /**
        * just for test.
        * @param args
        */
       public static void main(String[] args) {
           String content = getEnumContent();
           System.out.println(content);
           
           String[] testGrade = {"DEBUG", "INFO", "WARN", "ERROR", "TEST"};
           for (String tg : testGrade) {
               if (!ValidGrade.isValid(tg)) {
                   String message = String.format("%s is invalid.", tg);
                   System.out.println(message);
               }
           }
       }
   }

9. MessageTemplate.java

   package cn.sinobest.asj.log.util;
   /**
    * 消息模板.
    * @author lijinlong
    *
    */
   public class MessageTemplate {
       /** 无效的消息等级 */
       public static final String INVALID_GRADE = "无效的日志等级[%s]。服务支持的日志等级有：%s。";
       
       /** 无效的消息内容格式 */
       public static final String INVALID_FORMAT = "无效的日志内容格式:\n%s\n，请检查是否为JSON格式的字符串。";
   }

10. sun-jaxws.xml

    <?xml version="1.0" encoding="UTF-8"?>
    <endpoints xmlns="http://java.sun.com/xml/ns/jax-ws/ri/runtime"
        version="2.0">
        <endpoint name="defaultLog" implementation="cn.sinobest.asj.log.SALogImpl"
            url-pattern="/log.action" />
    </endpoints>

应用系统群

1. LogFilter.java

   package cn.sinobest.asj.log;
   import java.io.IOException;
   import java.net.MalformedURLException;
   import java.net.URL;
   import java.util.Date;
   import java.util.HashMap;
   import java.util.Map;
   import javax.servlet.Filter;
   import javax.servlet.FilterChain;
   import javax.servlet.FilterConfig;
   import javax.servlet.ServletException;
   import javax.servlet.ServletRequest;
   import javax.servlet.ServletResponse;
   import javax.servlet.http.HttpServletRequest;
   import org.apache.commons.logging.Log;
   import org.apache.commons.logging.LogFactory;
   import org.json.JSONObject;
   import cn.sinobest.asj.log.wsimport.ISALog;
   import cn.sinobest.asj.log.wsimport.InvalidFormatExceptioin_Exception;
   import cn.sinobest.asj.log.wsimport.InvalidGradeException_Exception;
   import cn.sinobest.asj.log.wsimport.SALogImplService;
   public class LogFilter implements Filter {
       static final Log log = LogFactory.getLog(LogFilter.class);
       static final String WSDL_LOCATION = "http://localhost:8080/logserv/log.action?wsdl";
       @Override
       public void destroy() {
       }
       @Override
       public void doFilter(ServletRequest request, ServletResponse response,
               FilterChain chain) throws IOException, ServletException {
           try {
               log(request);
           } catch (InvalidFormatExceptioin_Exception e) {
               e.printStackTrace();
           } catch (InvalidGradeException_Exception e) {
               e.printStackTrace();
           } finally {
               chain.doFilter(request, response);
           }
       }
       private void log(ServletRequest request) throws MalformedURLException,
               InvalidFormatExceptioin_Exception, InvalidGradeException_Exception {
           Map<String, Object> data = new HashMap<String, Object>();
           data.put("appid", "zfba");
           data.put("time", new Date());
           data.put("localAddr", request.getLocalAddr());
           data.put("localName", request.getLocalName());
           data.put("localPort", request.getLocalPort());
           data.put("remoteAddr", request.getRemoteAddr());
           data.put("remoteHost", request.getRemoteHost());
           data.put("remotePort", request.getRemotePort());
           // data.put("serverName", request.getServerName());
           // data.put("serverPort", request.getServerPort());
           HttpServletRequest hrequest = (HttpServletRequest) request;
           data.put("pathInfo", hrequest.getPathInfo());
           data.put("pathTranslated", hrequest.getPathTranslated());
           data.put("remoteUser", hrequest.getRemoteUser());
           data.put("requestURI", hrequest.getRequestURI());
           data.put("requestURL", hrequest.getRequestURL());
           data.put("servletPath", hrequest.getServletPath());
           JSONObject cont = new JSONObject(data);
           URL url = new URL(WSDL_LOCATION);
           SALogImplService ss = new SALogImplService(url);
           ISALog service = ss.getSALogImplPort();
           service.log("info", cont.toString());
       }
       @Override
       public void init(FilterConfig arg0) throws ServletException {
       }
   }

2. web.xml
   这里仅贴出新增的内容：

       <!-- 测试日志 -->
       <filter>
           <filter-name>log-filter</filter-name>
           <filter-class>cn.sinobest.asj.log.LogFilter</filter-class>
       </filter>
       <!-- 测试日志 -->
       <filter-mapping>
           <filter-name>log-filter</filter-name>
           <url-pattern>/*</url-pattern>
       </filter-mapping>

测试

1. 启动MongoDB
   参考《Flume学习应用：Java写日志数据到MongoDB》
2. 配置并启动Flume
   参考《Flume学习应用：Java写日志数据到MongoDB》
3. 启动日志服务
   参考《在web项目中发布jaxws》
4. 启动应用系统，并进行访问
5. 查看MongoDB数据库
   参考《Flume学习应用：Java写日志数据到MongoDB》

附录

1. Flume学习应用：Java写日志数据到MongoDB
   博客园：Flume学习应用：Java写日志数据到MongoDB
2. 在web项目中发布jaxws
   博客园：在web项目中发布jaxws
3. 基于wsimport生成代码的客户端
   博客园：基于wsimport生成代码的客户端