基于Jsoup的数据抓取

@Test//使用get请求
    public void test1() throws IOException {
        URL realUrl = new URL("http://www.baidu.com");
        // 打开和URL之间的连接  
        URLConnection conn = realUrl.openConnection();  
        // 设置通用的请求属性   请求头 关于http协议请求头内容可以查看我的另一篇博客:http://www.cnblogs.com/dougest/p/7130147.html
        conn.setRequestProperty("accept", "*/*");  
        conn.setRequestProperty("connection", "Keep-Alive");  
        conn.setRequestProperty("user-agent",  "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)");  
        // 建立实际的连接  
        conn.connect();  
        // 定义BufferedReader输入流来读取URL的响应  
        BufferedReader in = new BufferedReader(  
                new InputStreamReader(conn.getInputStream()));  
        String line;
        String result = "";
        while ((line = in.readLine()) != null) {  
            result += "\n" + line;  
        }
        System.out.println(result);//打印出百度的html
    }

　　@Test
    public void test2() throws IOException {
        //Jsoup内部封装了url请求/参数请求等,这里我们直接调用
        Document doc = Jsoup.connect("http://news.baidu.com/").get();
        System.out.println(doc.text());
    }

　　@Test
    public void test2() throws IOException {
        //Jsoup内部封装了http.net的一些方法,使用起来更简单
        //获取博客园首页文章信息
        Document doc = Jsoup.connect("https://www.cnblogs.com/").get();
        Elements elements = doc.getElementsByClass("titlelnk");
        for(Element e : elements) {
            System.out.println("标题==>"+e.text().trim());//获取标题
            System.out.println("超链接==>"+e.attr("href"));//获取超链接
        }
    }

WebClient webClient = new WebClient(BrowserVersion.CHROME);
        //支持JavaScript
        webClient.getOptions().setJavaScriptEnabled(true);//启用JS解释器，默认为true  
        webClient.getOptions().setCssEnabled(false);//禁用css支持 
        webClient.getOptions().setActiveXNative(false);
        webClient.getOptions().setCssEnabled(false);
        webClient.getOptions().setThrowExceptionOnScriptError(false);
        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
        webClient.getOptions().setTimeout(1*1000);
        webClient.getOptions().setUseInsecureSSL(true);
        HtmlPage rootPage = null;
        try {
            rootPage = webClient.getPage("https://www.*****.com");
        } catch (FailingHttpStatusCodeException e) {
            e.printStackTrace();
        } catch (MalformedURLException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
        //设置一个运行JavaScript的时间
        webClient.waitForBackgroundJavaScript(1*1000);
        String html = rootPage.asXml();
        Document doc = Jsoup.parse(html);