标签：mct erer ini java tap org ast 1.3 grok
#环境 centos 7.4 ,ELK 6 ,单节点
#服务端
Logstash 收集,过滤
Elasticsearch 存储,索引日志
Kibana 可视化
#客户端
filebeat 监控、转发,作为agent
filebeat-->Logstash-->Elasticsearch-->Kibana
#内核
echo ‘
* hard nofile 65536
* soft nofile 65536
* soft nproc 65536
* hard nproc 65536
‘>>/etc/security/limit.conf
echo ‘ vm.max_map_count = 262144
net.core.somaxconn=65535
net.ipv4.ip_forward = 1
‘>>/etc/sysctl.conf
sysctl -p
#关闭selinux、防火墙
systemctl stop firewalld.service
systemctl disable firewalld.service
firewall-cmd --state
sed -i ‘/^SELINUX=.*/c SELINUX=disabled‘ /etc/selinux/config
sed -i ‘s/^SELINUXTYPE=.*/SELINUXTYPE=disabled/g‘ /etc/selinux/config
grep --color=auto ‘^SELINUX‘ /etc/selinux/config
setenforce 0
#配置yum源
echo ‘
[elk-6]
name=elk-6
gpgcheck=1
enabled=1
‘>/etc/yum.repos.d/elk.repo
安装
yum install java-1.8.0-openjdk -y
yum install elasticsearch -y
yum install logstash -y
yum install kibana -y
elasticsearch配置
cp /etc/elasticsearch/elasticsearch.yml{,.bak}
echo ‘
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
cluster.name: ELK
node.name: elk.novalocal
network.host: 0.0.0.0
http.port: 9200
discovery.zen.ping.unicast.hosts: ["192.168.1.30:9300"]
discovery.zen.minimum_master_nodes: 1
‘>/etc/elasticsearch/elasticsearch.yml
systemctl daemon-reload
systemctl enable elasticsearch
systemctl restart elasticsearch
systemctl status elasticsearch
logstash配置
cp /etc/logstash/logstash.yml{,.bak}
echo ‘path.config: /etc/logstash/conf.d‘>>/etc/logstash/logstash.yml
#添加一个日志处理文件
#filebeat->logstash->elasticsearch
echo ‘
input {
#收集本地log#
file {
type => "logtest"
path => "/var/log/logtest.txt"
start_position => "beginning"
}
#filebeat客户端#
beats {
port => 5045
}
}
#筛选
#filter { }
output {
#标准输出，调试使用#
stdout {
codec => rubydebug { }
}
# 输出到es#
elasticsearch {
hosts => ["http://192.168.1.30:9200"]
index => "%{type}-%{+YYYY.MM.dd}"
}
}
‘>/etc/logstash/conf.d/logstash-01.conf
#检测配置
/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash-01.conf --config.test_and_exit
#生成测试
echo $(date +"%F-%T") log-test >>/var/log/logtest.txt
#启动，查看生成日志
nohup /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash-01.conf &
kibana配置
cp /etc/kibana/kibana.yml{,.bak}
echo ‘
server.port: 5601
server.host: "0.0.0.0"
# ES的url的一个ES节点#
elasticsearch.url: "http://192.168.1.30:9200"
kibana.index: ".kibana"
#kibana.defaultAppId: "home"
‘>/etc/kibana/kibana.yml
#启动
systemctl enable kibana
systemctl restart kibana
客户端安装
yum install -y filebeat
配置filebeat收集nginx日志
echo ‘
#filebeat#
filebeat.prospectors:
#nginx
- input_type: log
enable: yes
#tags: nginx-access
paths:
- /usr/local/nginx/logs/access.log
exclude_lines: ["^$"]
fields:
type: "nginx-access"
fields_under_root: true
output.logstash:
hosts: ["10.0.0.30:5044"]
#hosts: ["172.16.50.32:5044"]
#index: filebeat
‘>/etc/filebeat/filebeat.yml
启动
systemctl enable filebeat
systemctl restart filebeat
systemctl status filebeat
nginx默认日志格式
log_format main ‘$remote_addr - $remote_user [$time_local] "$request" ‘ ‘$status $body_bytes_sent "$http_referer" ‘ ‘"$http_user_agent" "$http_x_forwarded_for"‘;
创建nginx正则表达式(引用grok正则)
echo ‘#nginx-access
WZ ([^ ]*)
NGINXACCESS %{IP:remote_ip} \- \- \[%{HTTPDATE:timestamp}\] "%{WORD:method} %{WZ:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:status} %{NUMBER:bytes} %{QS:referer} %{QS:agent} %{QS:xforward}
‘>/etc/logstash/conf.d/nginx-access
重新生成logstash配置文件
echo ‘
input {
#收集本地log#
file {
type => "logtest"
path => "/var/log/logtest.txt"
start_position => "beginning"
}
#filebeat客户端#
beats {
port => 5044
}
}
# #筛选
filter {
# 如果是nginx访问日志
if ( [type] == "nginx-access" ) {
#按字段切割
grok {
patterns_dir=>"/etc/logstash/conf.d/nginx-access"
match => { "message" => "%{NGINXACCESS}" }
}
# 时间格式转换
date {
match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ]
}
# 删除不需要的字段
mutate {
remove_field => [ "offset", "@version", "beat", "input_type", "tags","id"]
}
}
}
output {
#标准输出，调试使用#
stdout {
codec => rubydebug { }
}
# 输出到es#
elasticsearch {
hosts => ["http://172.16.50.32:9200"]
index => "%{type}-%{+YYYY.MM.dd}"
}
}
‘>/etc/logstash/conf.d/logstash-01.conf
检测配置
/usr/share/logstash/bin/logstash -t -f /etc/logstash/conf.d/logstash-01.conf
调试logstash
#关闭
systemctl stop logstash
#在终端启动查看
/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash-01.conf
修改时区
timedatectl list-timezones |grep Shanghai #查找中国时区的完整名称
Asia/Shanghai
timedatectl set-timezone Asia/Shanghai #其他时区以此类推
ntpdate 0.asia.pool.ntp.org
ELK
标签：mct erer ini java tap org ast 1.3 grok
原文地址：https://www.cnblogs.com/xcbki/p/9978038.html
