logstash收集nginx访问日志

#直接yum安装：
[root@elk-node1 ~]# yum install nginx  -y
官方文档：http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format
#修改配置文件的日志格式：
vim /etc/nginx/nginx.conf  
#在http模块中添加
          log_format json ‘{"@timestamp":"$time_iso8601",‘
                           ‘"@version":"1",‘
                           ‘"client":"$remote_addr",‘
                           ‘"url":"$uri",‘
                           ‘"status":"$status",‘
                           ‘"domain":"$host",‘
                           ‘"host":"$server_addr",‘
                           ‘"size":$body_bytes_sent,‘
                           ‘"responsetime":$request_time,‘
                           ‘"referer": "$http_referer",‘
                           ‘"ua": "$http_user_agent"‘
               ‘}‘;
#在server模块中添加
access_log  /var/log/nginx/access_json.log  json;
#修改后的nginx.conf文件
[root@elk-node1 ~]# grep -Ev "#|^&" /etc/nginx/nginx.conf

user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;

include /usr/share/nginx/modules/*.conf;

events {
    worker_connections 1024;
}

http {
    log_format  main  ‘$remote_addr - $remote_user [$time_local] "$request" ‘
                      ‘$status $body_bytes_sent "$http_referer" ‘
                      ‘"$http_user_agent" "$http_x_forwarded_for"‘;
          log_format json ‘{"@timestamp":"$time_iso8601",‘
                           ‘"@version":"1",‘
                           ‘"client":"$remote_addr",‘
                           ‘"url":"$uri",‘
                           ‘"status":"$status",‘
                           ‘"domain":"$host",‘
                           ‘"host":"$server_addr",‘
                           ‘"size":$body_bytes_sent,‘
                           ‘"responsetime":$request_time,‘
                           ‘"referer": "$http_referer",‘
                           ‘"ua": "$http_user_agent"‘
               ‘}‘;
    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 2048;

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;

    include /etc/nginx/conf.d/*.conf;

    server {
        listen       80 default_server;
        listen       [::]:80 default_server;
        server_name  _;
        root         /usr/share/nginx/html;

        include /etc/nginx/default.d/*.conf;
    access_log  /var/log/nginx/access_json.log  json;

        location / {
        }

        error_page 404 /404.html;
            location = /40x.html {
        }

        error_page 500 502 503 504 /50x.html;
            location = /50x.html {
        }
    }


}
#启动：
[root@controller ~]# systemctl start nginx
root@elk-node1 ~]# ss -lntp|grep 80
LISTEN     0      511          *:80                       *:*                   users:(("nginx",pid=8045,fd=6),("nginx",pid=8044,fd=6),("nginx",pid=8043,fd=6))
LISTEN     0      511         :::80                      :::*                   users:(("nginx",pid=8045,fd=7),("nginx",pid=8044,fd=7),("nginx",pid=8043,fd=7))

#添加nginx日志格式到之前logstash的elk-log.yml
[root@elk-node1 ~]# cat /etc/logstash/conf.d/elk_log.conf
input {
    file {
      path => "/var/log/messages"
      type => "system"
      start_position => "beginning"
    }
    file {
       path => "/var/log/elasticsearch/hejianlai.log"
       type => "es-error"
       start_position => "beginning"
      codec => multiline {
          pattern => "^\["
          negate => true
          what => "previous"
        }
    }
       file {
       path => "/var/log/nginx/access_json.log"
       codec => json
       start_position => "beginning"
       type => "nginx-log"
    }
}
output {
    
    if [type] == "system"{
        elasticsearch {
           hosts => ["192.168.247.135:9200"]
           index => "systemlog-%{+YYYY.MM.dd}"
        }
    }
 
    if [type] == "es-error"{
        elasticsearch {
           hosts => ["192.168.247.135:9200"]
           index => "es-error-%{+YYYY.MM.dd}"
        }
    }
       if [type] == "nginx-log"{
        elasticsearch {
           hosts => ["192.168.247.135:9200"]
           index => "nginx-log-%{+YYYY.MM.dd}"
        }
    }
}