#!/bin/sh
if [ -r $1 ]; then
#Delete Baiduspider
baidu=$(grep -c ‘Baiduspider‘ $1)
if [ $baidu -gt 0 ]; then
sed -i ‘/Baiduspider/d‘ $1
echo "match Baiduspider $baidu line."
else
echo "Baiduspider not found."
fi
#Delete YandexBot
yandex=$(grep -c ‘YandexBot‘ $1)
if [ $yandex -gt 0 ]; then
sed -i ‘/YandexBot/d‘ $1
echo "match YandexBot $yandex line."
else
echo "YandexBot not found."
fi
#Delete bingbot
bing=$(grep -c ‘bingbot‘ $1)
if [ $bing -gt 0 ]; then
sed -i ‘/bingbot/d‘ $1
echo "match bingbot $bing line."
else
echo "bingbot not found."
fi
#Delete EasouSpider
easou=$(grep -c ‘EasouSpider‘ $1)
if [ $easou -gt 0 ]; then
sed -i ‘/EasouSpider/d‘ $1
echo "match EasouSpider $easou line."
else
echo "EasouSpider not found."
fi
#Delete Googlebot
Googlebot=$(grep -c ‘Googlebot‘ $1)
if [ $Googlebot -gt 0 ]; then
sed -i ‘/Googlebot/d‘ $1
echo "match Googlebot $Googlebot line."
else
echo "Googlebot not found."
fi
#Delete Mediapartners-Google
mgoogle=$(grep -c ‘Mediapartners-Google‘ $1)
if [ $mgoogle -gt 0 ]; then
sed -i ‘/Mediapartners-Google/d‘ $1
echo "match Mediapartners-Google $mgoogle line."
else
echo "Mediapartners-Google not found."
fi
#Delete Yahoo
yahoo=$(grep -c ‘Yahoo‘ $1)
if [ $yahoo -gt 0 ]; then
sed -i ‘/Yahoo/d‘ $1
echo "match Yahoo $yahoo line."
else
echo "Yahoo not found."
fi
#Delete YoudaoBot
youdao=$(grep -c ‘YoudaoBot‘ $1)
if [ $youdao -gt 0 ]; then
sed -i ‘/YoudaoBot/d‘ $1
echo "match YoudaoBot $youdao line."
else
echo "YoudaoBot not found."
fi
#Delete XoviBot
xovi=$(grep -c ‘XoviBot‘ $1)
if [ $xovi -gt 0 ]; then
sed -i ‘/XoviBot/d‘ $1
echo "match XoviBot $xovi line."
else
echo "XoviBot not found."
fi
#Delete MSNBot
msn=$(grep -c ‘MSNBot‘ $1)
if [ $msn -gt 0 ]; then
sed -i ‘/MSNBot/d‘ $1
echo "match MSNBot $msn line."
else
echo "MSNBot not found."
fi
#Delete Sogou web spider
sogou=$(grep -c ‘Sogou web spider‘ $1)
if [ $sogou -gt 0 ]; then
sed -i ‘/Sogou web spider/d‘ $1
echo "match Sogou web spider $sogou line."
else
echo "Sogou web spider not found."
fi
#Delete JikeSpider
jike=$(grep -c ‘JikeSpider‘ $1)
if [ $jike -gt 0 ]; then
sed -i ‘/JikeSpider/d‘ $1
echo "match JikeSpider $jike line."
else
echo "JikeSpider not found."
fi
#Delete proximic
proximic=$(grep -c ‘proximic‘ $1)
if [ $proximic -gt 0 ]; then
sed -i ‘/proximic/d‘ $1
echo "match proximic $proximic line."
else
echo "proximic not found."
fi
#Delete 360Spider
so360=$(grep -c ‘360Spider‘ $1)
if [ $so360 -gt 0 ]; then
sed -i ‘/360Spider/d‘ $1
echo "match 360Spider $so360 line."
else
echo "360Spider not found."
fi
#Delete 360spider-image
so360img=$(grep -c ‘360spider-image‘ $1)
if [ $so360img -gt 0 ]; then
sed -i ‘/360spider-image/d‘ $1
echo "match 360spider-image $so360img line."
else
echo "360spider-image not found."
fi
#Delete YYSpider
yy=$(grep -c ‘YYSpider‘ $1)
if [ $yy -gt 0 ]; then
sed -i ‘/YYSpider/d‘ $1
echo "match YYSpider $yy line."
else
echo "YYSpider not found."
fi
#Delete AhrefsBot
ahrefs=$(grep -c ‘AhrefsBot‘ $1)
if [ $ahrefs -gt 0 ]; then
sed -i ‘/AhrefsBot/d‘ $1
echo "match AhrefsBot $ahrefs line."
else
echo "AhrefsBot not found."
fi
#Delete msnbot
msnbot=$(grep -c ‘msnbot‘ $1)
if [ $msnbot -gt 0 ]; then
sed -i ‘/msnbot/d‘ $1
echo "match msnbot $msnbot line."
else
echo "msnbot not found."
fi
#Delete YisouSpider
yisou=$(grep -c ‘YisouSpider‘ $1)
if [ $yisou -gt 0 ]; then
sed -i ‘/YisouSpider/d‘ $1
echo "match YisouSpider $yisou line."
else
echo "YisouSpider not found."
fi
#Delete Facebot
facebot=$(grep -c ‘Facebot‘ $1)
if [ $facebot -gt 0 ]; then
sed -i ‘/Facebot/d‘ $1
echo "match Facebot $facebot line."
else
echo "Facebot not found."
fi
#Delete GrapeshotCrawler
grapeshot=$(grep -c ‘GrapeshotCrawler‘ $1)
if [ $grapeshot -gt 0 ]; then
sed -i ‘/GrapeshotCrawler/d‘ $1
echo "match GrapeshotCrawler $grapeshot line."
else
echo "GrapeshotCrawler not found."
fi
#Delete WeSEE
wesee=$(grep -c ‘WeSEE‘ $1)
if [ $wesee -gt 0 ]; then
sed -i ‘/WeSEE/d‘ $1
echo "match WeSEE $wesee line."
else
echo "WeSEE not found."
fi
#Delete Sogou inst spider
sogou_inst=$(grep -c ‘Sogou inst spider‘ $1)
if [ $sogou_inst -gt 0 ]; then
sed -i ‘/Sogou inst spider/d‘ $1
echo "match Sogou inst spider $sogou_inst line."
else
echo "Sogou inst spider not found."
fi
#Delete WebZIP/x.x
webzip=$(grep -c ‘www.spidersoft.com‘ $1)
if [ $webzip -gt 0 ]; then
sed -i ‘/www.spidersoft.com/d‘ $1
echo "match WebZIP/x.x $webzip line."
else
echo "WebZIP/x.x not found."
fi
#Delete baidu mobile spider
bms=$(grep -c ‘www.baidu.com\/search\/spider.html‘ $1)
if [ $bms -gt 0 ]; then
sed -i ‘/www.baidu.com\/search\/spider.html/d‘ $1
echo "match baidu mobile spider $bms line."
else
echo "baidu mobile spider not found."
fi
#Delete ChinasoSpider
chinaso=$(grep -c ‘ChinasoSpider‘ $1)
if [ $chinaso -gt 0 ]; then
sed -i ‘/ChinasoSpider/d‘ $1
echo "match ChinasoSpider $chinaso line."
else
echo "ChinasoSpider not found."
fi
#Delete www.xxx.com
xxx=$(grep -c ‘www.xxx.com‘ $1)
if [ $xxx -gt 0 ]; then
sed -i ‘/www.xxx.com/d‘ $1
echo "match www.xxx.com $xxx line."
else
echo "www.xxx.com not found."
fi
#Delete SEOENGWorldBot
seoeng=$(grep -c ‘SEOENGWorldBot‘ $1)
if [ $seoeng -gt 0 ]; then
sed -i ‘/SEOENGWorldBot/d‘ $1
echo "match SEOENGWorldBot $seoeng line."
else
echo "SEOENGWorldBot not found."
fi
#Delete Mail.RU_Bot
mailru=$(grep -c ‘Mail.RU_Bot‘ $1)
if [ $mailru -gt 0 ]; then
sed -i ‘/Mail.RU_Bot/d‘ $1
echo "match Mail.RU_Bot $mailru line."
else
echo "Mail.RU_Bot not found."
fi
#Delete Girafabot
gira=$(grep -c ‘Girafabot‘ $1)
if [ $gira -gt 0 ]; then
sed -i ‘/Girafabot/d‘ $1
echo "match Girafabot $gira line."
else
echo "Girafabot not found."
fi
#Delete DotBot
dotBot=$(grep -c ‘DotBot‘ $1)
if [ $dotBot -gt 0 ]; then
sed -i ‘/DotBot/d‘ $1
echo "match DotBot $dotBot line."
else
echo "DotBot not found."
fi
#Delete BLEXBot
BLEXBot=$(grep -c ‘BLEXBot‘ $1)
if [ $BLEXBot -gt 0 ]; then
sed -i ‘/BLEXBot/d‘ $1
echo "match BLEXBot $BLEXBot line."
else
echo "BLEXBot not found."
fi
else
echo "please, input your log file"
fi
将nginx日志中的蜘蛛记录删除掉,布布扣,bubuko.com
原文地址：http://my.oschina.net/jiangbianwanghai/blog/287746
