标签:http tab input nod war arch install rem img
修改nginx日志格式 log_format hanye ‘$proxy_add_x_forwarded_for $remote_user [$time_local] "$request" $http_host‘
‘[$body_bytes_sent] $request_body "$http_referer" "$http_user_agent" [$ssl_protocol] [$ssl_cipher]‘
‘[$request_time] [$status] [$upstream_status] [$upstream_response_time] [$upstream_addr]‘;
server {
listen 80;
server_name _;
access_log /data/wwwlogs/access_nginx.log hanye;
}
root@debian:~# ab -c 20 -n 20 http://192.168.1.252/
[root@elk-node01 wwwlogs]#cat /data/elk-services/logstash/patterns.d/nginx
NGUSERNAME [a-zA-Z\.\@\-\+_%]+
NGUSER %{NGUSERNAME}
NGINXACCESS %{IP:clent_ip} (?:-|%{USER:ident}) \[%{HTTPDATE:log_date}\] \"%{WORD:http_verb} (?:%{PATH:baseurl}\?%{NOTSPACE:params}(?: HTTP/%{NUMBER:http_version})?|%{DATA:raw_http_request})\" (%{IPORHOST:url_domain}|%{URIHOST:ur_domain}|-)\[(%{BASE16FLOAT:request_time}|-)\] %{NOTSPACE:request_body} %{QS:referrer_rul} %{GREEDYDATA:User_Agent} \[%{GREEDYDATA:ssl_protocol}\] \[(?:%{GREEDYDATA:ssl_cipher}|-)\]\[%{NUMBER:time_duration}\] \[%{NUMBER:http_status_code}\] \[(%{BASE10NUM:upstream_status}|-)\] \[(%{NUMBER:upstream_response_time}|-)\] \[(%{URIHOST:upstream_addr}|-)\]
[root@elk-node01 wwwlogs]# cat /data/elk-services/logstash/config/nginx_geoip.yml
input {
file {
path => "/data/wwwlogs/access_nginx.log"
type => "252nginx-access"
start_position => "beginning"
}
}
filter {
if [type] == "252nginx-access" {
grok {
patterns_dir => [ "/data/elk-services/logstash/patterns.d" ]
match => { "message" => "%{NGINXACCESS}" }
overwrite => [ "message" ]
}
geoip {
source => "clent_ip"
target => "geoip"
database => "/data/soft/GeoLite2-City_20190409/GeoLite2-City.mmdb"
}
useragent {
source => "User_Agent"
target => "userAgent"
}
urldecode {
all_fields => true
}
mutate {
gsub => ["User_Agent","[\"]",""] #将user_agent中的 " 换成空
convert => [ "response","integer" ]
convert => [ "body_bytes_sent","integer" ]
convert => [ "bytes_sent","integer" ]
convert => [ "upstream_response_time","float" ]
convert => [ "upstream_status","integer" ]
convert => [ "request_time","float" ]
convert => [ "port","integer" ]
}
date {
match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
}
}
}
output {
if [type] == "252nginx-access" {
elasticsearch {
hosts => ["192.168.1.252:9200"]
index => "logstash-nginx-access-252-%{+YYYY.MM.dd}"
}
}
}
[root@elk-node01 elasticsearch]#./bin/elasticsearch-plugin install ingest-geoip
[root@elk-node01 elasticsearch]# ./bin/elasticsearch-plugin install ingest-user-agent
[root@elk-node01 config]# ../bin/logstash -f nginx_geoip.yml
[root@elk-node01 config]# ab -c 20 -n 20 http://192.168.1.252/
####查看索引数据
标签:http tab input nod war arch install rem img
原文地址:https://blog.51cto.com/9025736/2377352