HtmlUnit模仿浏览器抓取数据（含ajax）

标签：

import java.io.IOException;
import java.net.MalformedURLException;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.SilentCssErrorHandler;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

public class WorldBankCrawl {


	public static void main(String[] args) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		    WebClient webClient = new WebClient(BrowserVersion.FIREFOX_24);
	        webClient.setCssErrorHandler(new SilentCssErrorHandler());
	        webClient.setAjaxController(new NicelyResynchronizingAjaxController());
	        webClient.getOptions().setCssEnabled(true);
	        webClient.getOptions().setRedirectEnabled(false);
	        webClient.getOptions().setAppletEnabled(false);
	        webClient.getOptions().setJavaScriptEnabled(true);
	        webClient.getOptions().setPopupBlockerEnabled(true);
	        webClient.getOptions().setTimeout(10000);
	        HtmlPage page = webClient.getPage("http://huaban.com/favorite/home/");
	        System.out.println(page.asXml());
	        webClient.closeAllWindows();
	}
}

HtmlUnit模仿浏览器抓取数据（含ajax）

标签：

原文地址：http://blog.csdn.net/hackcoder/article/details/43195297