码迷,mamicode.com
首页 > 编程语言 > 详细

网络爬虫Java实现抓取网页内容

时间:2016-06-05 17:03:42      阅读:428      评论:0      收藏:0      [点我收藏+]

标签:

package 抓取网页;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.methods.PostMethod;

public class RetrivePage {

private static HttpClient httpClient = new HttpClient();

public static void main(String[] args) {
//抓取猎兔的首页,并且输出出来
try {
RetrivePage.downloadPage("http://www.lietu.com");
} catch (HttpException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}

private static void downloadPage(String path) throws HttpException, IOException {
System.out.println("123123");
InputStream input = null;
OutputStream output = null;
//得到post方法
PostMethod postMethod = new PostMethod(path);
//设置post方法的参数
NameValuePair[] postData = new NameValuePair[2];
postData[0] = new NameValuePair("name","lietu");
postData[1] = new NameValuePair("password","*****");
//把参数添加到请求路径上去
postMethod.addParameters(postData);
//执行,返回状态码
int statusCode = httpClient.executeMethod(postMethod);
System.out.println(statusCode);
if (statusCode == HttpStatus.SC_OK) {
input = postMethod.getResponseBodyAsStream();
//得到文件的名字
String fileName = path.substring(path.lastIndexOf(‘/‘)+1);
//获得文件的输出流
System.out.println(fileName);
output = new FileOutputStream(fileName);

//输出到文件中
int tempByte = -1;
while ((tempByte = input.read()) > 0) {
output.write(tempByte);
}
//关闭资源
if (input != null) {
input.close();
}
if (output != null) {
output.close();
}
}
}
}

网络爬虫Java实现抓取网页内容

标签:

原文地址:http://www.cnblogs.com/airycode/p/5561015.html

(0)
(0)
   
举报
评论 一句话评论(0
登录后才能评论!
© 2014 mamicode.com 版权所有  联系我们:gaon5@hotmail.com
迷上了代码!