ELK收集日志到mysql

MariaDB [(none)]> create database elk character set utf8 collate utf8_bin;
Query OK, 1 row affected (0.00 sec)

MariaDB [(none)]> grant all privileges on elk.* to elk@‘192.168.20.%‘ identified by ‘123456‘;
Query OK, 0 rows affected (0.00 sec)

MariaDB [(none)]> flush privileges;
Query OK, 0 rows affected (0.00 sec)

[root@node1 ~]# yum install logstash-5.6.5.rpm -y
[root@node1 ~]# systemctl start logstash

[root@node1 ~]# yum install gem -y
[root@node1 ~]# gem -v
2.0.14.1
[root@node1 ~]# gem sources --add https://gems.ruby-china.org/ --remove https://rubygems.org/
https://gems.ruby-china.org/ added to sources
https://rubygems.org/ removed from sources
[root@node1 ~]# gem source list
*** CURRENT SOURCES ***

https://gems.ruby-china.org/

[root@node1 ~]# tar xf mysql-connector-java-5.1.45.tar.gz 
[root@node1 ~]# cd mysql-connector-java-5.1.45
[root@node1 mysql-connector-java-5.1.45]# mkdir -p /usr/share/logstash/vendor/jar/jdbc
[root@node1 mysql-connector-java-5.1.45]# cp mysql-connector-java-5.1.45-bin.jar  /usr/share/logstash/vendor/jar/jdbc/
[root@node1 ~]# chown -R logstash.logstash /usr/share/logstash/vendor/jar/jdbc/

log_format   access_log_json ‘{"host":"$http_x_real_ip","client_ip":"$remote_addr","log_time":"$time_iso8601","request":"$request","status":"$status","body_bytes_sent":"$body_bytes_sent","req_time":"$request_time","AgentVersion":"$http_user_agent"}‘;

access_log  /var/log/nginx/access.log  access_log_json;

nginx -t
nginx -s reload

MariaDB [elk]> create table nginx_log(host varchar(128),client_ip varchar(128),status int(4),req_time float(8,3),AgentVersion varchar(512), time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;

MariaDB [elk]> desc nginx_log;
+-------------+--------------+------+-----+-------------------+-------+
| Field       | Type         | Null | Key | Default           | Extra |
+-------------+--------------+------+-----+-------------------+-------+
| host        | varchar(128) | YES  |     | NULL              |       |
| client_ip   | varchar(128) | YES  |     | NULL              |       |
| status      | int(4)       | YES  |     | NULL              |       |
| req_time    | float(8,3)   | YES  |     | NULL              |       |
| AgentVersion | varchar(512) | YES  |     | NULL              |       |
| time        | timestamp    | NO   |     | CURRENT_TIMESTAMP |       |
+-------------+--------------+------+-----+-------------------+-------+
6 rows in set (0.00 sec)

[root@node1 ~]# cat /etc/logstash/conf.d/nginx_log.conf 
input{
  file{
    path => "/var/log/nginx/access.log"     # 指定文件
    start_position => "beginning"     # 从开始收集
    stat_interval => "2"            # 间隔时间为2s
    codec => "json"            # 使用json格式
}  

}

output{
  elasticsearch {
    hosts => ["192.168.20.60:9200"]
    index => "nginx-log-%{+YYYY.MM.dd}"
}

  jdbc{
    connection_string => "jdbc:mysql://192.168.20.61/elk?user=elk&password=123456&useUnicode=true&characterEncoding=UTF8"
    statement => ["insert into nginx_log(host,client_ip,status,req_time,AgentVersion) VALUES(?,?,?,?,?)", "host","client_ip","status","req_time","AgentVersion"]
}
}

[root@node1 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx_log.conf -t
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
Configuration OK