爬虫——博客实例

标签：

//Rule.java用于指定查询url,method,params
public class Rule
{
private String url;//链接
private String[] params;//参数集合
private String[] values;//参数的值
//对返回的HTML，第一希过滤所用的标签，先设置type
private String resultTagName;
/*设置resultTagName的类型，默认是ID
*CLASS/ID/SELECTION
*/
private int type=ID;
/*请求的类型，默认是get
* GET/POST
*/
private int requestMethod=GET;
public final static int GET=0;
public final static int POST=1;
public final static int class="0";
public final static int ID=1;
public final static int SELECTION=2;
public Rule()
{
}

public Rule(String url,String[] param,String[] values,String resultTagName,int type,int requestMethod)
{
super();
this.url=url;
this.params=params;
this.values=values;
this.resultTagName=resultTagName;
this.type=type;
this.requestMethod=requestMethod;
}

public String getUrl()
{
return url;
}
public void setUrl(String url)
{
this.url=url;
}
public String[] getParams()
{
return params;
}
public void setParams(String[] params)
{
this.params=params;
}
public String[] getValues()
{
return values;
}
public void setValues(String[] values)
{
this.values=values;
}
public String getResultTagName()
{
return resultTagName;
}
public void setResultTagName(String resultTagName)
{
this.resultTagName=resultTagName;
}
public int getType()
{
return type;
}
public void setType(int type)
{
this.type=type;
}
public int getRequestMethod()
{
return requestMethod;
}
public void setRequestMethod()
{
this.requestMethod=requestMethod;
}
}

//链接需要的数据对象
public class LinkTypeData {
private int id;
// 链接的地址
private String linkHref;
//链接标题
private String linkText;
//摘要
private String summary;
//内容
private String content;
public int getId()
{
return id;
}
public void setId(int id)
{
this.id=id;
}
public String getLinkHref()
{
return linkHref;
}
public void setLinkHref(String linkHref)
{
this.linkHref=linkHref;
}
public String getSummary()
{
return summary;
}
public void setSummary(String summary)
{
this.summary=summary;
}
public String getContent()
{
return content;
}
public void setContent(String content)
{
this.content=content;
}

}

import java.util.List;

//核心的查询类
public class ExtractService {
public static List<LinkTypeData> extract(Rule rule)//<>是泛型，里面指定了这个集合中存放的是什么数据
{
//对rule必要检验
validateRule(rule);
List<LinkTypeData> datas=new ArrayList<LinkTypeData>();
LinkTypeData data=null;
try
{
//解析rule
String url=rule.getUrl();
String[] params=rule.getParams();
String[] values=rule.getValues();
String resultTagName=rule.getResultTagName();
int type=rule.getType;
int requestType=rule.getRequestMethod();
connection conn=Jsoup.connect(url);//Jsoup.connect解析url网站地址
//设置查询参数
if(params!=null)
{
for(int i=0;i<params.length;i++)
{
conn.data(params[i],values[i]);
}
}
}
}
}

爬虫——博客实例

标签：

原文地址：http://www.cnblogs.com/sunshinewxz/p/4430355.html