HtmlUnit 开发网络爬虫（一）

import java.io.IOException;
import java.net.MalformedURLException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.Map;
import java.util.Queue;
import java.util.Set;

import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

//引用HtmlUnit
public class MyWalker {

    static int num = 0;//暂时用num进行限制
    
    final static WebClient webClient = new WebClient();
    static Queue<String> Q = new LinkedList<String>();//存放被抓取的Url
    static Set<String> linkUrl = new HashSet<String>();
    static Map<String,Boolean> flagMap = new HashMap<String,Boolean>();//标记Url是否访问过
    
    static{
        webClient.getOptions().setCssEnabled(false);
        webClient.getOptions().setJavaScriptEnabled(false);
    }
    
    private static String baseUrl = "http://xjasc.gov.cn";
    
    //拼凑出可再次访问的url, 这里的拼凑是不完整的
    private static String createUrl(String current, String u){
　　　　  return current + "/" + u;
    }
    
    //判断该链接是否合法
    private static boolean isLegal(String url){
        if(num > 100) return false;
        return true;
    }
    
    //页面是否坏掉
    private static boolean isBadUrl(String url){        
        return false;
    }
  
    private static  void bfs(){
        Q.offer(baseUrl);
        linkUrl.add(baseUrl);
        flagMap.put(baseUrl,false);
        while(!Q.isEmpty()){
            //得到当前的Url
            try {
                String tmpUrl = Q.poll();
                if(flagMap.get(tmpUrl)==false){//该结点没有访问过
                    flagMap.put(tmpUrl, true);//标记为已经访问
                    HtmlPage page = webClient.getPage(tmpUrl);
                    java.util.List<HtmlAnchor> achList=page.getAnchors();
                    for(HtmlAnchor ach:achList){            
                        String newUrl = createUrl(tmpUrl, ach.getHrefAttribute());            
                        if(isLegal(newUrl)){
                            System.out.println(newUrl);
                            num++;
                            linkUrl.add(newUrl);
                            flagMap.put(newUrl,false);
                            Q.offer(newUrl);
                        }                    
                    }        
                    
                }    
            } catch (FailingHttpStatusCodeException e) {    
                System.out.println(e.getStatusCode());
            } catch (MalformedURLException e) {
            } catch (IOException e) {
            }    
            
        }
        
    }

    public static void main(String[] args) {
        bfs();
    }
    
}