标签:tst empty 方法 nec tproxy throws user val 获取网页内容
package utils; import dao.PlatformDao; import okhttp3.RequestBody; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.NameValuePair; import org.apache.http.client.ResponseHandler; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.BasicResponseHandler; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.PrintWriter; import java.net.ConnectException; import java.net.SocketTimeoutException; import java.net.URL; import java.net.URLConnection; import java.util.*; /** * http请求器 * @author songmin */ public class SpiderHttpClient { private static final Logger log = LoggerFactory.getLogger(SpiderHttpClient.class); String proxyIP; Integer proxyPort; String responseParam = null; String url; String defaultEncoding = "gbk"; String contentEncoding = "UTF-8"; String chartSet = "UTF-8"; String userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36"; private static final String HEADER_REFERER = "Referer"; private static final String HEADER_COOKIE = "Cookie"; private static final String HEADER_USER_AGENT = "User-Agent"; Map<String, String> headerMap = new HashMap<String, String>(); Map<String, String> bodyMap = new HashMap<>(); public SpiderHttpClient(String url) { this.url = url; headerMap.put(HEADER_USER_AGENT, userAgent); } /** * 设置请求体 * @param key * @param value */ public void setBody(String key,String value){ bodyMap.put(key,value); } public void setProxyIP(String proxyIP) { if (proxyIP != null && !proxyIP.isEmpty()) { this.proxyIP = proxyIP; } } public void setProxyPort(Integer proxyPort) { if (proxyPort != null && proxyPort != 0) { this.proxyPort = proxyPort; } } /**设置字符格式*/ public void setChartSet(String chartSet){ this.chartSet = chartSet; } /** * 设置请求头 * * @param key * @param value */ public void setHeader(String key, String value) { headerMap.put(key, value); } /** * 设置referer * * @param referer */ public void setReferer(String referer) { setHeader(HEADER_REFERER, referer); } /** * 设置cookie * * @param cookie */ public void setCookie(String cookie) { setHeader(HEADER_COOKIE, cookie); } /** * 设置编码 */ public void setEncoding(String encoding) { this.defaultEncoding = encoding; } /** * 设置origin * @param origin */ public void setOrigin(String origin){ setHeader("Origin",origin); } public void setHost(String host){ setHeader("Host",host); } public void setContentType(String contentType){ setHeader("Content-Type",contentType); } public void setAcceptEncoding(String acceptEncoding){ setHeader("Accept-Encoding",acceptEncoding); } /** * 获取响应头 */ public String getResponseHeader(String key) throws IOException { this.responseParam = key; HttpResponse httpResponse = getResponse(url); String responseHeader = httpResponse.getResponseHeaderParam(); return responseHeader; } /** * 获取网页内容 * * @return */ public String getContent() throws IOException { HttpResponse httpResponse = getResponse(url); String html = httpResponse.getContent(); return html; } /** * 获取响应 * * @return */ public HttpResponse getResponse(String url) throws IOException { String content = null; //创建httpClient实例 CloseableHttpClient httpClient = HttpClients.createDefault(); //创建httpGet实例 HttpGet httpGet = new HttpGet(url); RequestConfig requestConfig = RequestConfig.custom() .setConnectTimeout(10000) .setSocketTimeout(10000) .setConnectionRequestTimeout(3000) .build(); //设置代理IP,设置连接超时时间 、 设置 请求读取数据的超时时间 、 设置从connect Manager获取Connection超时时间、 if (proxyIP != null && proxyPort != null && !proxyIP.isEmpty()) { HttpHost proxy = new HttpHost(proxyIP, proxyPort); requestConfig = RequestConfig.custom() .setProxy(proxy) .setConnectTimeout(10000) .setSocketTimeout(10000) .setConnectionRequestTimeout(3000) .build(); } Iterator<String> iterator = headerMap.keySet().iterator(); while (iterator.hasNext()) { String key = iterator.next(); httpGet.setHeader(key, headerMap.get(key)); } httpGet.setConfig(requestConfig); //设置请求头消息 CloseableHttpResponse response = httpClient.execute(httpGet); String responseHeader = ""; if (response != null) { HttpEntity entity = response.getEntity(); if (responseParam != null) { responseHeader = response.getFirstHeader(responseParam).getValue(); } if (entity != null) { content = EntityUtils.toString(entity, defaultEncoding); } } if (response != null) { response.close(); } if (httpClient != null) { httpClient.close(); } HttpResponse httpResponse = new HttpResponse(); httpResponse.setContent(content); httpResponse.setResponseHeaderParam(responseHeader); return httpResponse; } /** * 携带json发送post请求 * @param json 请求json * @return 返回值 */ public String sendJson(String json) { String returnValue = "这是默认返回值,接口调用失败"; CloseableHttpClient httpClient = HttpClients.createDefault(); ResponseHandler<String> responseHandler = new BasicResponseHandler(); try{ //第一步:创建HttpClient对象 httpClient = HttpClients.createDefault(); //第二步:创建httpPost对象 HttpPost httpPost = new HttpPost(url); Iterator<String> iterator = headerMap.keySet().iterator(); while (iterator.hasNext()) { String key = iterator.next(); httpPost.setHeader(key, headerMap.get(key)); } //第三步:给httpPost设置JSON格式的参数 StringEntity requestEntity = new StringEntity(json,chartSet); requestEntity.setContentEncoding(contentEncoding); httpPost.setEntity(requestEntity); //第四步:发送HttpPost请求,获取返回值 //调接口获取返回值时,必须用此方法 returnValue = httpClient.execute(httpPost,responseHandler); } catch(Exception e) { e.printStackTrace(); } finally { try { httpClient.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } //第五步:处理返回值 return returnValue; } /** * 发送post请求 * @return 请求结果 * @throws IOException */ public String sendHttpPost() throws IOException { CloseableHttpClient httpClient = HttpClients.createDefault(); ResponseHandler<String> responseHandler = new BasicResponseHandler(); HttpPost httpPost = new HttpPost(url); Iterator<String> iterator = headerMap.keySet().iterator(); while (iterator.hasNext()) { String key = iterator.next(); httpPost.setHeader(key, headerMap.get(key)); } // 创建参数队列 List<NameValuePair> nameValuePairs = new ArrayList<>(); for (String key : bodyMap.keySet()) { nameValuePairs.add(new BasicNameValuePair(key, bodyMap.get(key))); } try { httpPost.setEntity(new UrlEncodedFormEntity(nameValuePairs, "UTF-8")); } catch (Exception e) { e.printStackTrace(); } String result = httpClient.execute(httpPost,responseHandler); return result; } }
标签:tst empty 方法 nec tproxy throws user val 获取网页内容
原文地址:https://www.cnblogs.com/ushowtime/p/11676850.html