首页>代码>httpclientutil最简单最全的HTTP爬虫工具JAVA版>/httpclientutil-master/src/main/java/com/arronlong/httpclientutil/HttpClientUtil.java
package com.arronlong.httpclientutil; import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.NameValuePair; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpOptions; import org.apache.http.client.methods.HttpPatch; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.client.methods.HttpTrace; import org.apache.http.protocol.HttpContext; import org.apache.http.util.EntityUtils; import com.arronlong.httpclientutil.builder.HCB; import com.arronlong.httpclientutil.common.HttpConfig; import com.arronlong.httpclientutil.common.HttpMethods; import com.arronlong.httpclientutil.common.Utils; import com.arronlong.httpclientutil.exception.HttpProcessException; /** * 使用HttpClient模拟发送(http/https)请求 * * @author arron * @date 2015年11月4日 下午4:10:59 * @version 1.0 */ public class HttpClientUtil{ //默认采用的http协议的HttpClient对象 private static HttpClient client4HTTP; //默认采用的https协议的HttpClient对象 private static HttpClient client4HTTPS; static{ try { client4HTTP = HCB.custom().build(); client4HTTPS = HCB.custom().ssl().build(); } catch (HttpProcessException e) { Utils.errorException("创建https协议的HttpClient对象出错:{}", e); } } /** * 判定是否开启连接池、及url是http还是https <br> * 如果已开启连接池,则自动调用build方法,从连接池中获取client对象<br> * 否则,直接返回相应的默认client对象<br> * * @param config 请求参数配置 * @throws HttpProcessException */ private static void create(HttpConfig config) throws HttpProcessException { if(config.client()==null){//如果为空,设为默认client对象 if(config.url().toLowerCase().startsWith("https://")){ config.client(client4HTTPS); }else{ config.client(client4HTTP); } } } //-----------华----丽----分----割----线-------------- //-----------华----丽----分----割----线-------------- //-----------华----丽----分----割----线-------------- /** * 以Get方式,请求资源或服务 * * @param client client对象 * @param url 资源地址 * @param headers 请求头信息 * @param context http上下文,用于cookie操作 * @param encoding 编码 * @return 返回处理结果 * @throws HttpProcessException */ public static String get(HttpClient client, String url, Header[] headers, HttpContext context, String encoding) throws HttpProcessException { return get(HttpConfig.custom().client(client).url(url).headers(headers).context(context).encoding(encoding)); } /** * 以Get方式,请求资源或服务 * * @param config 请求参数配置 * @return * @throws HttpProcessException */ public static String get(HttpConfig config) throws HttpProcessException { return send(config.method(HttpMethods.GET)); } /** * 以Post方式,请求资源或服务 * * @param client client对象 * @param url 资源地址 * @param headers 请求头信息 * @param parasMap 请求参数 * @param context http上下文,用于cookie操作 * @param encoding 编码 * @return 返回处理结果 * @throws HttpProcessException */ public static String post(HttpClient client, String url, Header[] headers, Map<String,Object>parasMap, HttpContext context, String encoding) throws HttpProcessException { return post(HttpConfig.custom().client(client).url(url).headers(headers).map(parasMap).context(context).encoding(encoding)); } /** * 以Post方式,请求资源或服务 * * @param config 请求参数配置 * @return * @throws HttpProcessException */ public static String post(HttpConfig config) throws HttpProcessException { return send(config.method(HttpMethods.POST)); } /** * 以Put方式,请求资源或服务 * * @param client client对象 * @param url 资源地址 * @param parasMap 请求参数 * @param headers 请求头信息 * @param context http上下文,用于cookie操作 * @param encoding 编码 * @return 返回处理结果 * @throws HttpProcessException */ public static String put(HttpClient client, String url, Map<String,Object>parasMap,Header[] headers, HttpContext context,String encoding) throws HttpProcessException { return put(HttpConfig.custom().client(client).url(url).headers(headers).map(parasMap).context(context).encoding(encoding)); } /** * 以Put方式,请求资源或服务 * * @param config 请求参数配置 * @return * @throws HttpProcessException */ public static String put(HttpConfig config) throws HttpProcessException { return send(config.method(HttpMethods.PUT)); } /** * 以Delete方式,请求资源或服务 * * @param client client对象 * @param url 资源地址 * @param headers 请求头信息 * @param context http上下文,用于cookie操作 * @param encoding 编码 * @return 返回处理结果 * @throws HttpProcessException */ public static String delete(HttpClient client, String url, Header[] headers, HttpContext context,String encoding) throws HttpProcessException { return delete(HttpConfig.custom().client(client).url(url).headers(headers).context(context).encoding(encoding)); } /** * 以Delete方式,请求资源或服务 * * @param config 请求参数配置 * @return * @throws HttpProcessException */ public static String delete(HttpConfig config) throws HttpProcessException { return send(config.method(HttpMethods.DELETE)); } /** * 以Patch方式,请求资源或服务 * * @param client client对象 * @param url 资源地址 * @param parasMap 请求参数 * @param headers 请求头信息 * @param context http上下文,用于cookie操作 * @param encoding 编码 * @return 返回处理结果 * @throws HttpProcessException */ public static String patch(HttpClient client, String url, Map<String,Object>parasMap, Header[] headers, HttpContext context,String encoding) throws HttpProcessException { return patch(HttpConfig.custom().client(client).url(url).headers(headers).map(parasMap).context(context).encoding(encoding)); } /** * 以Patch方式,请求资源或服务 * * @param config 请求参数配置 * @return * @throws HttpProcessException */ public static String patch(HttpConfig config) throws HttpProcessException { return send(config.method(HttpMethods.PATCH)); } /** * 以Head方式,请求资源或服务 * * @param client client对象 * @param url 资源地址 * @param headers 请求头信息 * @param context http上下文,用于cookie操作 * @param encoding 编码 * @return 返回处理结果 * @throws HttpProcessException */ public static String head(HttpClient client, String url, Header[] headers, HttpContext context,String encoding) throws HttpProcessException { return head(HttpConfig.custom().client(client).url(url).headers(headers).context(context).encoding(encoding)); } /** * 以Head方式,请求资源或服务 * * @param config 请求参数配置 * @return * @throws HttpProcessException */ public static String head(HttpConfig config) throws HttpProcessException { return send(config.method(HttpMethods.HEAD)); } /** * 以Options方式,请求资源或服务 * * @param client client对象 * @param url 资源地址 * @param headers 请求头信息 * @param context http上下文,用于cookie操作 * @param encoding 编码 * @return 返回处理结果 * @throws HttpProcessException */ public static String options(HttpClient client, String url, Header[] headers, HttpContext context,String encoding) throws HttpProcessException { return options(HttpConfig.custom().client(client).url(url).headers(headers).context(context).encoding(encoding)); } /** * 以Options方式,请求资源或服务 * * @param config 请求参数配置 * @return * @throws HttpProcessException */ public static String options(HttpConfig config) throws HttpProcessException { return send(config.method(HttpMethods.OPTIONS)); } /** * 以Trace方式,请求资源或服务 * * @param client client对象 * @param url 资源地址 * @param headers 请求头信息 * @param context http上下文,用于cookie操作 * @param encoding 编码 * @return 返回处理结果 * @throws HttpProcessException */ public static String trace(HttpClient client, String url, Header[] headers, HttpContext context, String encoding) throws HttpProcessException { return trace(HttpConfig.custom().client(client).url(url).headers(headers).context(context).encoding(encoding)); } /** * 以Trace方式,请求资源或服务 * * @param config 请求参数配置 * @return * @throws HttpProcessException */ public static String trace(HttpConfig config) throws HttpProcessException { return send(config.method(HttpMethods.TRACE)); } /** * 下载文件 * * @param client client对象 * @param url 资源地址 * @param headers 请求头信息 * @param context http上下文,用于cookie操作 * @param out 输出流 * @return 返回处理结果 * @throws HttpProcessException */ public static OutputStream down(HttpClient client, String url, Header[] headers, HttpContext context, OutputStream out) throws HttpProcessException { return down(HttpConfig.custom().client(client).url(url).headers(headers).context(context).out(out)); } /** * 下载文件 * * @param config 请求参数配置 * @return 返回处理结果 * @throws HttpProcessException */ public static OutputStream down(HttpConfig config) throws HttpProcessException { return fmt2Stream(execute(config.method(HttpMethods.GET)), config.out()); } /** * 上传文件 * * @param client client对象 * @param url 资源地址 * @param headers 请求头信息 * @param context http上下文,用于cookie操作 * @return 返回处理结果 * @throws HttpProcessException */ public static String upload(HttpClient client, String url, Header[] headers, HttpContext context) throws HttpProcessException { return upload(HttpConfig.custom().client(client).url(url).headers(headers).context(context)); } /** * 上传文件 * * @param config 请求参数配置 * @return 返回处理结果 * @throws HttpProcessException */ public static String upload(HttpConfig config) throws HttpProcessException { if(config.method() != HttpMethods.POST && config.method() != HttpMethods.PUT){ config.method(HttpMethods.POST); } return send(config); } /** * 查看资源链接情况,返回状态码 * * @param client client对象 * @param url 资源地址 * @param headers 请求头信息 * @param context http上下文,用于cookie操作 * @return 返回处理结果 * @throws HttpProcessException */ public static int status(HttpClient client, String url, Header[] headers, HttpContext context, HttpMethods method) throws HttpProcessException { return status(HttpConfig.custom().client(client).url(url).headers(headers).context(context).method(method)); } /** * 查看资源链接情况,返回状态码 * * @param config 请求参数配置 * @return 返回处理结果 * @throws HttpProcessException */ public static int status(HttpConfig config) throws HttpProcessException { return fmt2Int(execute(config)); } //-----------华----丽----分----割----线-------------- //-----------华----丽----分----割----线-------------- //-----------华----丽----分----割----线-------------- /** * 请求资源或服务 * * @param config 请求参数配置 * @return * @throws HttpProcessException */ public static String send(HttpConfig config) throws HttpProcessException { return fmt2String(execute(config), config.outenc()); } /** * 请求资源或服务 * * @param config 请求参数配置 * @return 返回HttpResponse对象 * @throws HttpProcessException */ private static HttpResponse execute(HttpConfig config) throws HttpProcessException { create(config);//获取链接 HttpResponse resp = null; try { //创建请求对象 HttpRequestBase request = getRequest(config.url(), config.method()); //设置header信息 request.setHeaders(config.headers()); //判断是否支持设置entity(仅HttpPost、HttpPut、HttpPatch支持) if(HttpEntityEnclosingRequestBase.class.isAssignableFrom(request.getClass())){ List<NameValuePair> nvps = new ArrayList<NameValuePair>(); //检测url中是否存在参数 config.url(Utils.checkHasParas(config.url(), nvps, config.inenc())); //装填参数 HttpEntity entity = Utils.map2HttpEntity(nvps, config.map(), config.inenc()); //设置参数到请求对象中 ((HttpEntityEnclosingRequestBase)request).setEntity(entity); Utils.info("请求地址:"+config.url()); if(nvps.size()>0){ Utils.info("请求参数:"+nvps.toString()); } if(config.json()!=null){ Utils.info("请求参数:"+config.json()); } }else{ int idx = config.url().indexOf("?"); Utils.info("请求地址:"+config.url().substring(0, (idx>0 ? idx : config.url().length()))); if(idx>0){ Utils.info("请求参数:"+config.url().substring(idx+1)); } } //执行请求操作,并拿到结果(同步阻塞) resp = (config.context()==null)?config.client().execute(request) : config.client().execute(request, config.context()) ; if(config.isReturnRespHeaders()){ //获取所有response的header信息 config.headers(resp.getAllHeaders()); } //获取结果实体 return resp; } catch (IOException e) { throw new HttpProcessException(e); } } //-----------华----丽----分----割----线-------------- //-----------华----丽----分----割----线-------------- //-----------华----丽----分----割----线-------------- /** * 转化为字符串 * * @param resp 响应对象 * @param encoding 编码 * @return * @throws HttpProcessException */ private static String fmt2String(HttpResponse resp, String encoding) throws HttpProcessException { String body = ""; try { if (resp.getEntity() != null) { // 按指定编码转换结果实体为String类型 body = EntityUtils.toString(resp.getEntity(), encoding); Utils.info(body); }else{//有可能是head请求 body =resp.getStatusLine().toString(); } EntityUtils.consume(resp.getEntity()); } catch (IOException e) { throw new HttpProcessException(e); }finally{ close(resp); } return body; } /** * 转化为数字 * * @param resp 响应对象 * @return * @throws HttpProcessException */ private static int fmt2Int(HttpResponse resp) throws HttpProcessException { int statusCode; try { statusCode = resp.getStatusLine().getStatusCode(); EntityUtils.consume(resp.getEntity()); } catch (IOException e) { throw new HttpProcessException(e); }finally{ close(resp); } return statusCode; } /** * 转化为流 * * @param resp 响应对象 * @param out 输出流 * @return * @throws HttpProcessException */ public static OutputStream fmt2Stream(HttpResponse resp, OutputStream out) throws HttpProcessException { try { resp.getEntity().writeTo(out); EntityUtils.consume(resp.getEntity()); } catch (IOException e) { throw new HttpProcessException(e); }finally{ close(resp); } return out; } /** * 根据请求方法名,获取request对象 * * @param url 资源地址 * @param method 请求方式 * @return */ private static HttpRequestBase getRequest(String url, HttpMethods method) { HttpRequestBase request = null; switch (method.getCode()) { case 0:// HttpGet request = new HttpGet(url); break; case 1:// HttpPost request = new HttpPost(url); break; case 2:// HttpHead request = new HttpHead(url); break; case 3:// HttpPut request = new HttpPut(url); break; case 4:// HttpDelete request = new HttpDelete(url); break; case 5:// HttpTrace request = new HttpTrace(url); break; case 6:// HttpPatch request = new HttpPatch(url); break; case 7:// HttpOptions request = new HttpOptions(url); break; default: request = new HttpPost(url); break; } return request; } /** * 尝试关闭response * * @param resp HttpResponse对象 */ private static void close(HttpResponse resp) { try { if(resp == null) return; //如果CloseableHttpResponse 是resp的父类,则支持关闭 if(CloseableHttpResponse.class.isAssignableFrom(resp.getClass())){ ((CloseableHttpResponse)resp).close(); } } catch (IOException e) { Utils.exception(e); } } }

mq13947193109 LV19
2024年10月9日
mugege123 LV6
2023年5月30日
Micwang LV1
2023年5月7日
212600 LV7
2022年9月6日
唐僧洗头爱飘柔 LV22
2022年7月19日
DeepLoveDeep LV2
2021年11月23日
四月 LV1
2021年9月17日
谈笑之间 LV6
2021年7月24日
我是helloworld LV23
2021年5月23日
我是风 LV13
2021年4月10日