标签：elk日志分析平台
本文记录了三个配置文件：
第一个：all.conf 通过一个配置文件，配置输入输出，实例；
第二个：shipper.conf配置logstash收集日志内容到redis里；
第三个：indexer.conf配置logstash从redis里读取日志内容输出到Elasticsearch里。
第二个跟第三个配置解耦收集日志
ELK 解耦
logstash ---------->redis ---------->logstash -------->elasticsearch----------->kibana
收集 消息队列 分析 存储 显示
192.168.1.87 192.168.1.87 192.168.1.88 192.168.1.87 192.168.1.87
all.conf 通过一个配置文件，配置输入输出，实例；
[root@y0 ~]# cat all.conf
input {
file {
path => "/var/log/messages"
type => "system"
start_position => "beginning"
}
file {
path => "/var/log/nginx/access_json.log"
codec => "json"
type => "nginx_log"
start_position => "beginning"
}
file {
path => "/var/log/elasticsearch/Mint.log"
type => "es-error"
start_position => "beginning"
codec => multiline {
pattern => "^\["
negate => true
what => "previous"
}
}
syslog {
type => "system-syslog"
host => "192.168.1.87"
port => "514"
}
}
output {
if [type] == "system" {
elasticsearch {
hosts => ["192.168.1.87:9200"]
index => "system-%{+YYYY.MM.dd}"
}
}
if [type] == "es-error" {
elasticsearch {
hosts => ["192.168.1.87:9200"]
index => "es-error-%{+YYYY.MM.dd}"
}
}
if [type] == "nginx_log" {
elasticsearch {
hosts => ["192.168.1.87:9200"]
index => "nginx_log-%{+YYYY.MM.dd}"
}
}
if [type] == "system-syslog" {
elasticsearch {
hosts => ["192.168.1.87:9200"]
index => "system-syslog-%{+YYYY.MM.dd}"
}
}
}
shipper.conf配置logstash收集日志内容到redis里；
[root@y0 ~]# cat shipper.conf
input {
file {
path => "/var/log/messages"
type => "system"
start_position => "beginning"
}
file {
path => "/var/log/nginx/access_json.log"
codec => "json"
type => "nginx_log"
start_position => "beginning"
}
file {
path => "/var/log/elasticsearch/Mint.log"
type => "es-error"
start_position => "beginning"
codec => multiline {
pattern => "^\["
negate => true
what => "previous"
}
}
syslog {
type => "system-syslog"
host => "192.168.1.87"
port => "514"
}
}
output {
if [type] == "system" {
redis {
host => "192.168.1.87 "
port => "6379"
db => "6"
data_type => "list"
key => "system"
}
}
if [type] == "es-error" {
redis {
host => "192.168.1.87 "
port => "6379"
db => "6"
data_type => "list"
key => "es-error"
}
}
if [type] == "nginx_log" {
redis {
host => "192.168.1.87 "
port => "6379"
db => "6"
data_type => "list"
key => "nginx_log"
}
}
if [type] == "system-syslog" {
redis {
host => "192.168.1.87 "
port => "6379"
db => "6"
data_type => "list"
key => "system-syslog"
}
}
}
indexer.conf 配置logstash从redis里读取日志内容输出到Elasticsearch里。
[root@test01 ~]# cat indexer.conf
input {
redis {
type => "system"
host => "192.168.1.87 "
port => "6379"
db => "6"
data_type => "list"
key => "system"
}
redis {
type => "es-error"
host => "192.168.1.87 "
port => "6379"
db => "6"
data_type => "list"
key => "es-error"
}
redis {
type => "nginx_log"
host => "192.168.1.87 "
port => "6379"
db => "6"
data_type => "list"
key => "nginx_log"
}
redis {
type => "system-syslog"
host => "192.168.1.87 "
port => "6379"
db => "6"
data_type => "list"
key => "system-syslog"
}
}
output {
if [type] == "system" {
elasticsearch {
hosts => ["192.168.1.87:9200"]
index => "system-%{+YYYY.MM.dd}"
}
}
if [type] == "es-error" {
elasticsearch {
hosts => ["192.168.1.87:9200"]
index => "es-error-%{+YYYY.MM.dd}"
}
}
if [type] == "nginx_log" {
elasticsearch {
hosts => ["192.168.1.87:9200"]
index => "nginx_log-%{+YYYY.MM.dd}"
}
}
if [type] == "system-syslog" {
elasticsearch {
hosts => ["192.168.1.87:9200"]
index => "system-syslog-%{+YYYY.MM.dd}"
}
}
}本文出自 “share,open source” 博客，请务必保留此出处http://liqilong2010.blog.51cto.com/3029053/1946598
标签：elk日志分析平台
原文地址：http://liqilong2010.blog.51cto.com/3029053/1946598
