com.panie 项目开发随笔_爬虫初识（2017.2.7）

package com.panie.modules.crawl.webcollector.example;

import org.jsoup.nodes.Document;

import com.panie.modules.crawl.webcollector.model.CrawlDatums;
import com.panie.modules.crawl.webcollector.model.Page;
import com.panie.modules.crawl.webcollector.plugin.berkeley.BreadthCrawler;

public class NewsCrawler extends BreadthCrawler
{
    public NewsCrawler(String crawlPath, boolean autoParse)
    {
        super(crawlPath, autoParse);
        /*start page*/
        this.addSeed("http://news.hfut.edu.cn/list-1-1.html");

        /*fetch url like http://news.hfut.edu.cn/show-xxxxxxhtml*/
        this.addRegex("http://news.hfut.edu.cn/show-.*html");
        /*do not fetch jpg|png|gif*/
        this.addRegex("-.*\\.(jpg|png|gif).*");
        /*do not fetch url contains #*/
        this.addRegex("-.*#.*");
    }
    
    @Override
    public void visit(Page page, CrawlDatums next)
    {
        String url = page.getUrl();
        /*if page is news page*/
        if (page.matchUrl("http://news.hfut.edu.cn/show-.*html")) {
            /*we use jsoup to parse page*/
            Document doc = page.getDoc();

            /*extract title and content of news by css selector*/
            String title = page.select("div[id=Article]>h2").first().text();
            String content = page.select("div#artibody", 0).text();

            System.out.println("URL:\n" + url);
            System.out.println("title:\n" + title);
            System.out.println("content:\n" + content);

            /*If you want to add urls to crawl,add them to nextLink*/
            /*WebCollector automatically filters links that have been fetched before*/
            /*If autoParse is true and the link you add to nextLinks does not match the regex rules,the link will also been filtered.*/
            //next.add("http://xxxxxx.com");
        }
    }
    
    public static void main(String[] args) throws Exception
    {
        NewsCrawler crawler = new NewsCrawler("crawl", true);
        //crawler.addSeed("http://news.hfut.edu.cn/.*");
        crawler.setThreads(50);
        crawler.setTopN(100);
        //crawler.setResumable(true);
        /*start crawl with depth of 4*/
        crawler.start(4);
    }
}