标签：百度网页 shell baiduspider
#!/bin/bash
#desc: this scripts for baidunews-spider
#date:2014.02.25
#testd in CentOS 5.9 x86_64
#saved in /usr/local/bin/baidu-web.sh
#written by coralzd@gmail.com www.zjyxh.com
dt=`date -d "yesterday" +%m%d`
if [ $1x != x ] ;then
if [ -e $1 ] ;then
grep -i "Baiduspider/2.0" $1 > baiduspider-${dt}.txt
num=`cat baiduspider-${dt}.txt|wc -l`
echo "baiduspider number is ${num},file is baidu-${dt}.txt"
cat baiduspider-${dt}.txt|awk ‘{print $7}‘|sort |uniq -c|sort -r >`ls ${1}|cut -c 1-10`-${dt}.txt
echo "$1 was done"
else
echo "$1 not exsist!"
fi
else
echo "usage: $0 file_path"
fi本次用shell分析百度网页蜘蛛跟百度新闻蜘蛛一个方法，无非就是把关键词由baiduspider-news换为baiduspider/2.0。
本文出自 “崔晓辉的博客” 博客，请务必保留此出处http://coralzd.blog.51cto.com/90341/1590956
标签：百度网页 shell baiduspider
原文地址：http://coralzd.blog.51cto.com/90341/1590956
