01 logstash应用详解
配置环境：
node3 192.168.1.133 CentOS Linux release 7.2
node4 192.168.1.134 CentOS Linux release 7.2
[root@node4 ~]# cd /etc/logstash/conf.d/
[root@node4 conf.d]# vim filesample.conf
input {
file {
path => ["/var/log/messages"]
type => "system"
start_position => "beginning"
}
}
output {
stdout {
codec => rubydebug
}
}
[root@node4 conf.d]# logstash -f filesample.conf --configtest
Configuration OK
[root@node3 ~]# rpm -ivh epel-release-latest-7.noarch.rpm
[root@node3 ~]# yum -y install collectd
[root@node3 ~]# vim /etc/collectd.conf
修改
#Hostname "localhost"
为
Hostname "node3"
修改
#LoadPlugin df
为
LoadPlugin df #监控磁盘
修改
#LoadPlugin network
为
LoadPlugin network
在<Plugin netlink>程序端后添加
<Plugin network>
<Server "192.168.1.134" "25826">
</Server>
</Plugin>
[root@node3 ~]# systemctl start collectd.service
[root@node4 conf.d]# vim udpsample.conf
input {
udp {
port => 25826
codec => collectd {}
type => "collectd"
}
}
output {
stdout {
codec => rubydebug
}
}
[root@node4 conf.d]# logstash -f udpsample.conf --configtest
Configuration OK
[root@node4 conf.d]# logstash -f udpsample.conf
[root@node4 conf.d]# yum -y install httpd
[root@node4 conf.d]# systemctl start httpd.service
[root@node4 conf.d]# vim groksample.conf
input {
stdin {}
}
filter {
grok {
match => { "message" => "%{IP:clientip} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}" }
}
}
output {
stdout {
codec => rubydebug
}
}
[root@node4 conf.d]# logstash -f groksample.conf --configtest
Configuration OK
[root@node4 conf.d]# logstash -f groksample.conf
Logstash startup completed
1.1.1.1 GET /index.html 30 0.23
{
"message" => "1.1.1.1 GET /index.html 30 0.23",
"@version" => "1",
"@timestamp" => "2017-01-03T13:37:24.978Z",
"host" => "node4",
"clientip" => "1.1.1.1",
"method" => "GET",
"request" => "/index.html",
"bytes" => "30",
"duration" => "0.23"
}
[root@node4 conf.d]# vim apachelogsample.conf
input {
file {
path => ["/var/log/httpd/access_log"]
type => "apachelog"
start_position => "beginning"
}
}
filter {
grok {
match => { "message" => "%{COMBINEDAPACHELOG}" }
}
}
output {
stdout {
codec => rubydebug
}
}
[root@node4 conf.d]# logstash -f apachelogsample.conf --configtest
Configuration OK
[root@node4 conf.d]# vim /opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-0.3.0/patterns/grok-patterns
在末尾添加
# nginx Logs
NGUSERNAME [a-zA-Z\.\@\-\+_%]+
NGUSER %{NGUSERNAME}
NGINXACCESS %{IPORHOST:clientip} - %{NOTSPACE:remote_user} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{OS:agent} %{NOTSPACE:http_x_forwarded_for}
[root@node4 conf.d]# systemctl stop httpd.service
[root@node4 conf.d]# yum -y install nginx
[root@node4 conf.d]# systemctl start nginx.service
[root@node4 conf.d]# cd /var/log/nginx/
[root@node4 nginx]# ls
access.log error.log
[root@node4 nginx]# tail access.log
192.168.1.204 - - [03/Jan/2017:22:18:03 +0800] "GET / HTTP/1.1" 200 3700 "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.3; WOW64; Trident/7.0; .NET4.0E; .NET4.0C)" "-"
192.168.1.204 - - [03/Jan/2017:22:18:03 +0800] "GET /nginx-logo.png HTTP/1.1" 200 368 "http://192.168.1.134/" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.3; WOW64; Trident/7.0; .NET4.0E; .NET4.0C)" "-"
192.168.1.204 - - [03/Jan/2017:22:18:03 +0800] "GET /poweredby.png HTTP/1.1" 200 2811 "http://192.168.1.134/" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.3; WOW64; Trident/7.0; .NET4.0E; .NET4.0C)" "-"
[root@node4 conf.d]# cd -
[root@node4 conf.d]# cp apachelogsample.conf nginxlogsample.conf
[root@node4 conf.d]# vim nginxlogsample.conf
input {
file {
path => ["/var/log/nginx/access.log"]
type => "nginxlog"
start_position => "beginning"
}
}
filter {
grok {
match => { "message" => "%{NGINXACCESS}" }
}
}
output {
stdout {
codec => rubydebug
}
}
[root@node4 conf.d]# logstash -f nginxlogsample.conf
02 ELK Stack
[root@node4 ~]# yum install redis
[root@node4 ~]# vim /etc/redis.conf
修改
bind 127.0.0.1
为
bind 0.0.0.0
[root@node4 ~]# systemctl start redis.service
[root@node4 ~]# redis-cli
127.0.0.1:6379> help
redis-cli 2.8.19
Type: "help @<group>" to get a list of commands in <group>
"help <command>" for help on <command>
"help <tab>" to get a list of possible help topics
"quit" to exit
[root@node4 ~]# cd /etc/logstash/conf.d/
[root@node4 conf.d]# cp nginxlogsample.conf nglogredissample.conf
[root@node4 conf.d]# vim nglogredissample.conf
input {
file {
path => ["/var/log/nginx/access.log"]
type => "nginxlog"
start_position => "beginning"
}
}
filter {
grok {
match => { "message" => "%{NGINXACCESS}" }
}
}
output {
redis {
port => 6379
host => ["127.0.0.1"]
data_type => "list"
key => "logstash-%[type]"
}
}
[root@node4 conf.d]# logstash -f nglogredissample.conf --configtest
Configuration OK
[root@node3 ~]# vim /etc/profile.d/java.sh
export JAVA_HOME=/usr
[root@node3 ~]# yum install -y logstash-1.5.4-1.noarch.rpm
[root@node3 ~]# cd /etc/logstash/conf.d/
[root@node3 conf.d]# vim server.conf
input {
redis {
port => "6370"
host => "192.168.1.134"
data_type => "list"
key => "logstash-nginxlog"
}
}
output {
stdout {
codec => rubydebug
}
}
[root@node3 conf.d]# vim /etc/profile.d/logstash.sh
export PATH=/opt/logstash/bin:$PATH
[root@node3 conf.d]# . /etc/profile.d/logstash.sh
[root@node3 conf.d]# logstash -f server.conf --configtest
Configuration OK
[root@node1 ~]# yum makecache
[root@node1 ~]# yum install java-1.7.0-openjdk-devel.x86_64
[root@node1 ~]# vim /etc/profile.d/java.sh
export JAVA_HOME=/usr
[root@node1 ~]# yum install elasticsearch-1.7.2.noarch.rpm -y
[root@node1 ~]# vim /etc/elasticsearch/elasticsearch.yml
修改
#cluster.name: elasticsearch
为
cluster.name: loges
修改
#node.name: "Franz Kafka"
为
node.name: "node1"
[root@node1 ~]# systemctl daemon-reload
[root@node1 ~]# systemctl start elasticsearch.service
[root@node1 ~]# /usr/share/elasticsearch/bin/plugin -i bigedsk -u file:///root/bigdesk-latest.zip
[root@node1 ~]# tar xf kibana-4.1.2-linux-x64.tar.gz -C /usr/local/
[root@node1 ~]# cd /usr/local/
[root@node1 local]# ln -s kibana-4.1.2-linux-x64/ kibana
[root@node1 local]# cd kibana
[root@node1 kibana]# ls
bin config LICENSE.txt node plugins README.txt src
[root@node1 kibana]# cd config/
[root@node1 config]# vim kibana.yml
修改
elasticsearch_url: "http://localhost:9200"
为
elasticsearch_url: "http://192.168.1.131:9200"
#启动
[root@node1 kibana]# /usr/local/kibana/bin/kibana
[root@node3 conf.d]# vim server.conf
input {
redis {
port => "6370"
host => "192.168.1.134"
data_type => "list"
key => "logstash-nginxlog"
}
}
output {
elasticsearch {
cluster => "loges"
index => "logstash-%{+YYYY.MM.dd}"
}
}
[root@node3 conf.d]# logstash -f server.conf --configtest
Configuration OK
[root@node3 conf.d]# logstash -f server.conf
[root@node1 ~]# curl -XGET ‘localhost:9200/_cat/indices‘
yellow open .kibana 1 1 1 0 2.4kb 2.4kb
该节视频到71:55(65382)由于错误太多无法继续进行
本文出自 “追梦” 博客，请务必保留此出处http://sihua.blog.51cto.com/377227/1889869
原文地址：http://sihua.blog.51cto.com/377227/1889869
