1.统计单个IP 的访问
1）.awk
awk ‘{++count[$1]};END {for (i in count) print i,count[i]}‘ nginx-access_page.xywy.log | sort -nr -k 2 | more
2）. awk + sort + uniq
awk ‘{print $1}’ nginx-access_page.xywy.log | sort | uniq -c | sort -nr
2.统计IP段的访问
1）awk
awk ‘{split($1,ip,"."); net=ip[1]"."ip[2]"."ip[3] ; ++count[net]};END {for (i in count) print i ".0/24",count[i]} ‘ /data/logs/nginx/nginx-access_z.xywy.com.log | sort -nr -k 2 | more
2.统计流量带宽
awk ‘{total_flow+=$11}END{print "total_pv:",NR ; print "total_flow:",total_flow/ 1024 /1024,"M" ; print "total_bandwidth:" , total_flow / 1024 /1024 /86400 * 8,"Mbps"}‘ /data/logs/cut-log/20160311/test.admin.ads.xywy.com-access_log
带宽保留两位小数
awk ‘{total_flow+=$10}END{print "total_pv:",NR ; print "total_flow:",total_flow/ 1024 /1024,"M" ; printf "total_bandwidth:" ; printf "%.2f" ,total_flow / 1024 /1024 /86400 * 8 ; print " Mbps"}‘ /data/logs/cut-log/20160311/test.admin.ads.xywy.com-access_log
统计每小时带宽、访问量
for i in {00..23} ; do echo "2016:$i:"; grep "2016:$i" /data/logs/cut-log/20160311/test.admin.ads.xywy.com-access_log | awk ‘{total_flow+=$10}END{print "total_pv:",NR ; print "total_flow:",total_flow/ 1024 /1024,"M" ; printf "total_bandwidth:" ; printf "%.2f" ,total_flow / 1024 /1024 /3600 * 8 ; print " Mbps"}‘ ; done
统计状态码的数量及百分比
for i in {6..9}; do echo 2016032$i ; awk ‘{++count[$10]};END {print "Total:",NR ;for (i in count) print i,count[i]}‘ 2016032$i/3g.club.xywy.com-access_log | sort -nr -k 2 ; done ?
或者
grep -oP ‘HTTP/1.." \d+ ‘ /data/logs/nginx/www.xywy.com-access_log | cut -d ‘ ‘ -f2 | sort | uniq -c
统计每小时访问量：
for i in `seq -f ‘%02g‘ 0 23` ; do echo -ne "2016:$i: "; grep -w "2016:$i" ./20160702/page.xywy.com-access.log | wc -l ; done
统计所有日志下，指定时间段访问量最高的IP
for i in `ls` ; do awk ‘/02:00:00/,/04:00:00/ {++count[$2]} ;END {for (s in count) if (count[s] > 2000) {print $1,s,count[s]}}‘ $i | sort -n -k 2 -r | head -5 ; done for i in `ls` ; do awk ‘/02\/Nov\/2016:02:00:00/,/02\/Nov\/2016:04:00:00/ {++count[$2]} ;END {for (s in count) if (count[s] > 3000) {print $1,s,count[s]}}‘ $i | sort -n -k 2 -r | head -5 ; done
3. 奇偶行合并
[fuzengjie@Mac ~/Downloads]$ seq 6 ‘1 2 3 4 5 6 [fuzengjie@Mac ~/Downloads]$ seq 6 | sed ‘N;s/\n//‘ 12 34 56
或者
[fuzengjie@Mac ~/Downloads]$ seq 6 1 2 3 4 5 6 [fuzengjie@Mac ~/Downloads]$ seq 6 | awk ‘(ORS=(i=!i)?"":RS)||1‘ 12 34 56
系统连接状态篇：
1.查看TCP连接状态
netstat -ant | awk ‘{print $6}‘ | sort | uniq -c | sort -rn
netstat -n | awk ‘/^tcp/ {++S[$NF]};END {for(a in S) print a, S[a]}‘ 或
netstat -n | awk ‘/^tcp/ {++state[$NF]}; END {for(key in state) print key,"\t",state[key]}‘
netstat -n | awk ‘/^tcp/ {++arr[$NF]};END {for(k in arr) print k,"t",arr[k]}‘
netstat -n |awk ‘/^tcp/ {print $NF}‘|sort|uniq -c|sort -rn
netstat -ant | awk ‘{print $NF}‘ | grep -v ‘[a-z]‘ | sort | uniq -c
3.用tcpdump嗅探80端口的访问看看谁最高
tcpdump -i eth0 -tnn dst port 80 -c 1000 | awk -F"." ‘{print $1"."$2"."$3"."$4}‘ | sort | uniq -c | sort -nr |head -20
4.查找较多time_wait连接
netstat -n|grep TIME_WAIT|awk ‘{print $5}‘|sort|uniq -c|sort -rn|head -n20
5.找查较多的SYN连接
netstat -an | grep SYN | awk ‘{print $5}‘ | awk -F: ‘{print $1}‘ | sort | uniq -c | sort -nr | more
6.根据端口列进程
netstat -ntlp | grep 80 | awk ‘{print $7}‘ | cut -d/ -f1
/usr/sbin/tcpdump -i eth0 -l -s 0 -w - dst port 80 | strings | grep -i user-agent | grep -i -E ‘bot|crawler|slurp|spider‘
网站日分析2(Squid篇）按域统计流量
zcat squid_access.log.tar.gz| awk ‘{print $10,$7}‘ |awk ‘BEGIN{FS="[ /]"}{trfc[$4]+=$1}END{for(domain in trfc){printf "%st%dn",domain,trfc[domain]}}‘
数据库篇
1.查看数据库执行的sql
/usr/sbin/tcpdump -i eth0 -s 0 -l -w - dst port 3306 | strings | egrep -i ‘SELECT|UPDATE|DELETE|INSERT|SET|COMMIT|ROLLBACK|CREATE|DROP|ALTER|CALL‘
系统Debug分析篇
1.调试命令
strace -p pid
2.跟踪指定进程的PID
gdb -p pid
awk ‘ {a=substr($2,1,3); b=substr($4,1,3); if ($a == $b ) print $1,$2,$3,$4}‘ ne_02.txt >> ne_02_01.txt
awk ‘ {a=substr($2,1,3); b=substr($4,1,3); if ($a != $b ) print $1,$2,$3,$4}‘ ne_02.txt >> ne_02_02.txt
awk ‘{split($2,a,"."); n=a[1]a[2]a[3];split($4,b,".");m=b[1]b[2]b[3] ; if((n == m))print $0 }‘ ne_02.txt
sed -n ‘/03\/Nov\/2015:12/,/03\/Nov\/2015:18/p‘ /data/logs/nginx/nginx-access_p.xywy.log | wc -l
sed -i ‘s/\(XYWYSRV_REDIS[1-9]\?_HOST_\?R\?\).*/\1 "172.16.207.27";/g‘
