引入依赖
<dependency><groupId>org.apache.httpcomponents</groupId><artifactId>httpclient</artifactId><version>4.5.13</version></dependency>
参数配置
连接池参数:
| 配置 | 简介 | 默认值 |
|---|---|---|
| maxConnTotal | 最大连接数 | 20 |
| maxConnPerRoute | 分配给同一个route(路由)最大的并发数。 举例来说,我们使用HttpClient的实现来分别请求 www.baidu.com 的资源和 www.bing.com 的资源那么他就会产生两个route |
2 |
| connTimeToLive | 连接池中connection的存活时间 | 永久有效 |
| maxIdleTime | 最大空闲时间 | 如果没有指定maxIdleTime的话,但是有设置evictExpiredConnections的话,默认是10秒 |
RequestConfig常用参数:
| 重要参数 | 默认值 | 特殊值(0) | ||
|---|---|---|---|---|
超时参数 |
socketTimeout | 指客户端从服务器读取数据的timeout,超出后会抛出SocketTimeOutException | -1,未定义使用默认值 | 无限超时 |
| connectTimeout | 客户端和服务器建立连接的timeout,就是http请求的三个阶段,一:建立连接;二:数据传送;三,断开连接。超时后会ConnectionTimeOutException | -1,未定义使用默认值 | 无限超时 | |
| connectionRequestTimeout | 从连接池获取连接的timeout。 HttpClient中的要用连接时尝试从连接池中获取,若是在等待了一定的时间后还没有获取到可用连接(比如连接池中没有空闲连接了)则会抛出获取连接超时异常。 |
-1,未定义使用默认值 | 无限超时 |
高并发下性能优化-http连接池
首先,明确两点:
1.http连接池不是万能的,过多的长连接会占用服务器资源,导致其他服务受阻
2.http连接池只适用于请求是经常访问同一主机(或同一个接口)的情况下
3.并发数不高的情况下资源利用率低下
使用http连接池的优点:
1.复用http连接,省去了tcp的3次握手和4次挥手的时间,极大降低请求响应的时间
2.自动管理tcp连接,不用人为地释放/创建连接
使用http连接池的大致流程 :
1.创建PoolingHttpClientConnectionManager实例
2.给manager设置参数
3.给manager设置重试策略
4.给manager设置连接管理策略
5.开启监控线程,及时关闭被服务器单向断开的连接
6.构建HttpClient实例
7.创建HttpPost/HttpGet实例,并设置参数
8.获取响应,做适当的处理
9.将用完的连接放回连接池
使用方式:
java原生使用方式:
package com.powerX.httpClient;import java.io.BufferedReader;import java.io.IOException;import java.io.InputStream;import java.io.InputStreamReader;import java.io.OutputStream;import java.net.HttpURLConnection;import java.net.MalformedURLException;import java.net.URL;public class HttpClient {public static String doGet(String httpurl) {HttpURLConnection connection = null;InputStream is = null;BufferedReader br = null;String result = null;// 返回结果字符串try {// 创建远程url连接对象URL url = new URL(httpurl);// 通过远程url连接对象打开一个连接,强转成httpURLConnection类connection = (HttpURLConnection) url.openConnection();// 设置连接方式:getconnection.setRequestMethod("GET");// 设置连接主机服务器的超时时间:15000毫秒connection.setConnectTimeout(15000);// 设置读取远程返回的数据时间:60000毫秒connection.setReadTimeout(60000);// 发送请求connection.connect();// 通过connection连接,获取输入流if (connection.getResponseCode() == 200) {is = connection.getInputStream();// 封装输入流is,并指定字符集br = new BufferedReader(new InputStreamReader(is, "UTF-8"));// 存放数据StringBuffer sbf = new StringBuffer();String temp = null;while ((temp = br.readLine()) != null) {sbf.append(temp);sbf.append("\r\n");}result = sbf.toString();}} catch (MalformedURLException e) {e.printStackTrace();} catch (IOException e) {e.printStackTrace();} finally {// 关闭资源if (null != br) {try {br.close();} catch (IOException e) {e.printStackTrace();}}if (null != is) {try {is.close();} catch (IOException e) {e.printStackTrace();}}connection.disconnect();// 关闭远程连接}return result;}public static String doPost(String httpUrl, String param) {HttpURLConnection connection = null;InputStream is = null;OutputStream os = null;BufferedReader br = null;String result = null;try {URL url = new URL(httpUrl);// 通过远程url连接对象打开连接connection = (HttpURLConnection) url.openConnection();// 设置连接请求方式connection.setRequestMethod("POST");// 设置连接主机服务器超时时间:15000毫秒connection.setConnectTimeout(15000);// 设置读取主机服务器返回数据超时时间:60000毫秒connection.setReadTimeout(60000);// 默认值为:false,当向远程服务器传送数据/写数据时,需要设置为trueconnection.setDoOutput(true);// 默认值为:true,当前向远程服务读取数据时,设置为true,该参数可有可无connection.setDoInput(true);// 设置传入参数的格式:请求参数应该是 name1=value1&name2=value2 的形式。connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");// 设置鉴权信息:Authorization: Bearer da3efcbf-0845-4fe3-8aba-ee040be542c0connection.setRequestProperty("Authorization", "Bearer da3efcbf-0845-4fe3-8aba-ee040be542c0");// 通过连接对象获取一个输出流os = connection.getOutputStream();// 通过输出流对象将参数写出去/传输出去,它是通过字节数组写出的os.write(param.getBytes());// 通过连接对象获取一个输入流,向远程读取if (connection.getResponseCode() == 200) {is = connection.getInputStream();// 对输入流对象进行包装:charset根据工作项目组的要求来设置br = new BufferedReader(new InputStreamReader(is, "UTF-8"));StringBuffer sbf = new StringBuffer();String temp = null;// 循环遍历一行一行读取数据while ((temp = br.readLine()) != null) {sbf.append(temp);sbf.append("\r\n");}result = sbf.toString();}} catch (MalformedURLException e) {e.printStackTrace();} catch (IOException e) {e.printStackTrace();} finally {// 关闭资源if (null != br) {try {br.close();} catch (IOException e) {e.printStackTrace();}}if (null != os) {try {os.close();} catch (IOException e) {e.printStackTrace();}}if (null != is) {try {is.close();} catch (IOException e) {e.printStackTrace();}}// 断开与远程地址url的连接connection.disconnect();}return result;}}
apache httpClient4.5使用方式:
package com.powerX.httpClient;import java.io.IOException;import java.io.UnsupportedEncodingException;import java.util.ArrayList;import java.util.Iterator;import java.util.List;import java.util.Map;import java.util.Map.Entry;import java.util.Set;import org.apache.http.HttpEntity;import org.apache.http.NameValuePair;import org.apache.http.client.ClientProtocolException;import org.apache.http.client.config.RequestConfig;import org.apache.http.client.entity.UrlEncodedFormEntity;import org.apache.http.client.methods.CloseableHttpResponse;import org.apache.http.client.methods.HttpGet;import org.apache.http.client.methods.HttpPost;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.impl.client.HttpClients;import org.apache.http.message.BasicNameValuePair;import org.apache.http.util.EntityUtils;public class HttpClient4 {public static String doGet(String url) {CloseableHttpClient httpClient = null;CloseableHttpResponse response = null;String result = "";try {// 通过址默认配置创建一个httpClient实例httpClient = HttpClients.createDefault();// 创建httpGet远程连接实例HttpGet httpGet = new HttpGet(url);// 设置请求头信息,鉴权httpGet.setHeader("Authorization", "Bearer da3efcbf-0845-4fe3-8aba-ee040be542c0");// 设置配置请求参数RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(35000)// 连接主机服务超时时间.setConnectionRequestTimeout(35000)// 请求超时时间.setSocketTimeout(60000)// 数据读取超时时间.build();// 为httpGet实例设置配置httpGet.setConfig(requestConfig);// 执行get请求得到返回对象response = httpClient.execute(httpGet);// 通过返回对象获取返回数据HttpEntity entity = response.getEntity();// 通过EntityUtils中的toString方法将结果转换为字符串result = EntityUtils.toString(entity);} catch (ClientProtocolException e) {e.printStackTrace();} catch (IOException e) {e.printStackTrace();} finally {// 关闭资源if (null != response) {try {response.close();} catch (IOException e) {e.printStackTrace();}}if (null != httpClient) {try {httpClient.close();} catch (IOException e) {e.printStackTrace();}}}return result;}public static String doPost(String url, Map<String, Object> paramMap) {CloseableHttpClient httpClient = null;CloseableHttpResponse httpResponse = null;String result = "";// 创建httpClient实例httpClient = HttpClients.createDefault();// 创建httpPost远程连接实例HttpPost httpPost = new HttpPost(url);// 配置请求参数实例RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(35000)// 设置连接主机服务超时时间.setConnectionRequestTimeout(35000)// 设置连接请求超时时间.setSocketTimeout(60000)// 设置读取数据连接超时时间.build();// 为httpPost实例设置配置httpPost.setConfig(requestConfig);// 设置请求头httpPost.addHeader("Content-Type", "application/x-www-form-urlencoded");// 封装post请求参数if (null != paramMap && paramMap.size() > 0) {List<NameValuePair> nvps = new ArrayList<NameValuePair>();// 通过map集成entrySet方法获取entitySet<Entry<String, Object>> entrySet = paramMap.entrySet();// 循环遍历,获取迭代器Iterator<Entry<String, Object>> iterator = entrySet.iterator();while (iterator.hasNext()) {Entry<String, Object> mapEntry = iterator.next();nvps.add(new BasicNameValuePair(mapEntry.getKey(), mapEntry.getValue().toString()));}// 为httpPost设置封装好的请求参数try {httpPost.setEntity(new UrlEncodedFormEntity(nvps, "UTF-8"));} catch (UnsupportedEncodingException e) {e.printStackTrace();}}try {// httpClient对象执行post请求,并返回响应参数对象httpResponse = httpClient.execute(httpPost);// 从响应对象中获取响应内容HttpEntity entity = httpResponse.getEntity();result = EntityUtils.toString(entity);} catch (ClientProtocolException e) {e.printStackTrace();} catch (IOException e) {e.printStackTrace();} finally {// 关闭资源if (null != httpResponse) {try {httpResponse.close();} catch (IOException e) {e.printStackTrace();}}if (null != httpClient) {try {httpClient.close();} catch (IOException e) {e.printStackTrace();}}}return result;}}
springboot使用方式:
连接池方式:
统一配置参数、设置默认值:
import lombok.Data;import lombok.experimental.Accessors;import org.apache.commons.lang3.ObjectUtils;import java.io.Serializable;/*** @author yaol*/@Data@Accessors(chain = true)public class HttpClientProperties implements Serializable {/*** 唯一标识, 日志辅助使用*/private String flag;/*** 代理ip, null 默认不使用代理*/private String httpProxyHost;/*** 代理端口,null 默认不使用代理*/private Integer httpProxyPort;/*** 从连接池获取连接的timeout。 / millisecond*/private Integer connectionRequestTimeout;/*** 客户端和服务器建立连接的timeout / millisecond*/private Integer connectTimeout;/*** 客户端从服务器读取数据的timeout / millisecond*/private Integer readTimeout;/*** 最大空闲连接时间 / millisecond*/private Integer maxIdleTime;/*** 服务最大连接数*/private Integer maxConnTotal;/*** 分配给同一个route(路由)最大的并发数*/private Integer maxConnPerRoute;/*** 是否需要启用证书验证*/private Boolean needSslValidation;/*** ssl签名证书路径*/private String sslCertPath;/*** ssl签名证书密码*/private String sslCertPassword;/*** 异常重试次数(null or <0: 不会重实)*/private Integer retryTimes;/*** 异常重试间隔(null or <=0: 直接重试,不做间隔) / millisecond*/private Integer retryIntervalTime;/*** 默认值设置*/public void buildDefault() {this.connectTimeout = getConnectTimeout();this.readTimeout = getReadTimeout();this.connectionRequestTimeout = getConnectionRequestTimeout();this.maxIdleTime = getMaxIdleTime();this.maxConnTotal = getMaxConnTotal();this.maxConnPerRoute = getMaxConnPerRoute();this.needSslValidation = getNeedSslValidation();this.retryTimes = getRetryTimes();this.retryIntervalTime = getRetryIntervalTime();}public Integer getConnectTimeout() {return ObjectUtils.defaultIfNull(this.connectTimeout, 10 * 1000);}public Integer getReadTimeout() {return ObjectUtils.defaultIfNull(this.readTimeout, 10 * 1000);}public Integer getConnectionRequestTimeout() {return ObjectUtils.defaultIfNull(this.connectionRequestTimeout, 3 * 1000);}public Integer getMaxIdleTime() {return ObjectUtils.defaultIfNull(this.maxIdleTime, 20 * 1000);}public Integer getMaxConnTotal() {return ObjectUtils.defaultIfNull(this.maxConnTotal, 200);}public Integer getMaxConnPerRoute() {return ObjectUtils.defaultIfNull(this.maxConnPerRoute, 100);}public Boolean getNeedSslValidation() {return ObjectUtils.defaultIfNull(this.needSslValidation, false);}public Integer getRetryTimes() {return ObjectUtils.defaultIfNull(this.retryTimes, 0);}public Integer getRetryIntervalTime() {return ObjectUtils.defaultIfNull(this.retryIntervalTime, 100);}}
ClientHttpRequestFactory工厂类工具类:
import com.alibaba.fastjson.JSON;import com.pingxx.common.consts.LogConsts;import com.pingxx.common.encryption.CertUtil;import com.pingxx.common.enums.LogTypeEnum;import lombok.extern.slf4j.Slf4j;import org.apache.commons.lang3.StringUtils;import org.apache.http.HttpHost;import org.apache.http.conn.ssl.NoopHostnameVerifier;import org.apache.http.conn.ssl.SSLConnectionSocketFactory;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.impl.client.HttpClientBuilder;import org.apache.http.impl.client.HttpClients;import org.apache.http.ssl.SSLContextBuilder;import org.apache.http.ssl.SSLContexts;import org.apache.http.ssl.TrustStrategy;import org.springframework.http.client.ClientHttpRequestFactory;import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;import javax.net.ssl.SSLContext;import java.security.GeneralSecurityException;import java.security.KeyStore;import java.util.concurrent.TimeUnit;import static net.logstash.logback.marker.Markers.append;/*** 统一ClientHttpRequestFactory* @author yaol*/@Slf4jpublic class ClientHttpRequestFactoryUtil {/*** 统一ClientHttpRequestFactory* @param httpClientProperties* @return*/public static ClientHttpRequestFactory requestFactory(HttpClientProperties httpClientProperties) {log.info("{} httpClientProperties: {}", httpClientProperties.getFlag(), JSON.toJSONString(httpClientProperties));try {HttpClientBuilder httpClientBuilder = HttpClients.custom();if (Boolean.TRUE.equals(httpClientProperties.getNeedSslValidation()) &&StringUtils.isNotBlank(httpClientProperties.getSslCertPath())) {//设置证书KeyStore keyStore = CertUtil.getKeyStore(httpClientProperties.getSslCertPath(),httpClientProperties.getSslCertPassword(), CertUtil.KEY_STORE_TYPE_PKCS12);if (keyStore != null) {SSLContextBuilder sslContextBuilder = SSLContexts.custom();try {sslContextBuilder.loadKeyMaterial(keyStore, httpClientProperties.getSslCertPassword().toCharArray()).loadTrustMaterial(keyStore, (chain, authType) -> true);SSLContext sslContext = sslContextBuilder.loadTrustMaterial(keyStore, (chain, authType) -> true).build();SSLConnectionSocketFactory connectionSocketFactory = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);httpClientBuilder.setSSLSocketFactory(connectionSocketFactory);} catch (GeneralSecurityException e) {log.error(append(LogConsts.LOG_TYPE, LogTypeEnum.EXCEPTION),"init SSLContext error:{}", httpClientProperties.getFlag(), e);throw new RuntimeException("init SSLContext error");}} else {log.error("HttpClient need ssl cert, but keyStore is null:{}", httpClientProperties.getFlag());throw new RuntimeException("HttpClient need ssl cert, but keyStore is null");}} else {//证书绕过TrustStrategy acceptingTrustStrategy = (x509Certificates, authType) -> true;SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, acceptingTrustStrategy).build();SSLConnectionSocketFactory connectionSocketFactory = new SSLConnectionSocketFactory(sslContext,new NoopHostnameVerifier());httpClientBuilder.setSSLSocketFactory(connectionSocketFactory);}//代理if (StringUtils.isNotBlank(httpClientProperties.getHttpProxyHost()) && httpClientProperties.getHttpProxyPort() != null) {httpClientBuilder.setProxy(new HttpHost(httpClientProperties.getHttpProxyHost(), httpClientProperties.getHttpProxyPort(), "http"));}// 只有io异常才会触发重试HttpRequestRetryHandler handler = new HttpRequestRetryHandler() {@Overridepublic boolean retryRequest(IOException exception, int curRetryCount, HttpContext context) {// curRetryCount 每一次都会递增,从1开始if (curRetryCount > retryTimes) {return false;}try {//重试延迟if (retryTimes > 0 && retryIntervalTime > 0 ) {Thread.sleep(curRetryCount * retryIntervalTime);}} catch (InterruptedException e) {throw new RuntimeException(e);}//重试场景if (exception instanceof ConnectTimeoutException|| exception instanceof NoHttpResponseException|| exception instanceof ConnectException// || exception instanceof UnknownHostException) {return true;}HttpClientContext clientContext = HttpClientContext.adapt(context);org.apache.http.HttpRequest request = clientContext.getRequest();boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);if (idempotent) {// 如果请求被认为是幂等的,那么就重试。即重复执行不影响程序其他效果的return true;}return false;}};//重试handlerhttpClientBuilder.setRetryHandler(handler);//最大连接数httpClientBuilder.setMaxConnTotal(httpClientProperties.getMaxConnTotal());//分配给同一个route(路由)最大的并发数httpClientBuilder.setMaxConnPerRoute(httpClientProperties.getMaxConnPerRoute());//最大空闲连接时间httpClientBuilder.evictExpiredConnections().evictIdleConnections(httpClientProperties.getMaxIdleTime(), TimeUnit.MILLISECONDS);HttpComponentsClientHttpRequestFactory requestFactory = new HttpComponentsClientHttpRequestFactory();//从连接池获取连接的timeout。requestFactory.setConnectionRequestTimeout(httpClientProperties.getConnectionRequestTimeout());//客户端和服务器建立连接的timeoutrequestFactory.setConnectTimeout(httpClientProperties.getConnectTimeout());//指客户端从服务器读取数据的timeoutrequestFactory.setReadTimeout(httpClientProperties.getReadTimeout());CloseableHttpClient httpClient = httpClientBuilder.build();requestFactory.setHttpClient(httpClient);return requestFactory;} catch (Exception e) {log.error("new ClientHttpRequestFactory exception: {}", httpClientProperties.getFlag(), e);throw new RuntimeException("new ClientHttpRequestFactory exception", e);}}}
获取nacos配置、初始化restTemplate:
import com.pingxx.common.util.http.ClientHttpRequestFactoryUtil;import com.pingxx.common.util.http.HttpClientProperties;import lombok.Getter;import lombok.Setter;import lombok.extern.slf4j.Slf4j;import org.springframework.boot.context.properties.ConfigurationProperties;import org.springframework.boot.context.properties.EnableConfigurationProperties;import org.springframework.boot.web.client.RestTemplateBuilder;import org.springframework.context.annotation.Bean;import org.springframework.context.annotation.Configuration;import org.springframework.http.client.ClientHttpRequestFactory;import org.springframework.web.client.RestTemplate;@Getter@Setter@Slf4j@EnableConfigurationProperties@Configuration@ConfigurationProperties(prefix = "http-client")public class HttpClientConfig {private String httpProxyHost;private Integer httpProxyPort;private Integer connectionRequestTimeout;private Integer connectTimeout;private Integer readTimeout;private Integer maxIdleTime;private Integer maxConnTotal;private Integer maxConnPerRoute;@Beanpublic ClientHttpRequestFactory requestFactory() {HttpClientProperties httpClientProperties = new HttpClientProperties().setHttpProxyHost(httpProxyHost).setHttpProxyPort(httpProxyPort).setConnectionRequestTimeout(connectionRequestTimeout).setConnectTimeout(connectTimeout).setReadTimeout(readTimeout).setMaxIdleTime(maxIdleTime).setMaxConnTotal(maxConnTotal).setMaxConnPerRoute(maxConnPerRoute).setFlag("common");httpClientProperties.buildDefault();return ClientHttpRequestFactoryUtil.requestFactory(httpClientProperties);}@Beanpublic RestTemplate restTemplate(RestTemplateBuilder builder) {return builder.requestFactory(this::requestFactory).build();}}
发起请求工具类:
package com.example.boothttp.http;import lombok.extern.slf4j.Slf4j;import org.slf4j.Marker;import org.springframework.http.*;import org.springframework.stereotype.Component;import org.springframework.web.client.RestTemplate;import static net.logstash.logback.marker.Markers.append;/*** @author yaol*/@Slf4j@Componentpublic class HttpRequestHelper {public HttpResp send(HttpReq reqArgs, RestTemplate restTemplate) {String url = reqArgs.getUrl();HttpHeaders reqHeaders = reqArgs.getReqHeaders();String reqBody = reqArgs.getReqBody();if (reqArgs.getReqHeaders() == null) {reqHeaders = new HttpHeaders();}//设置默认请求头reqHeaders.setContentType(MediaType.APPLICATION_JSON);reqHeaders.add(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE);HttpEntity<String> requestEntity = new HttpEntity<>(reqBody, reqHeaders);long requestStartAt = System.currentTimeMillis();String responseBody = "";int statusCode = 0;HttpHeaders responseHeaders = null;long costMs;try {ResponseEntity<String> responseEntity = restTemplate.postForEntity(url, requestEntity, String.class);statusCode = responseEntity.getStatusCodeValue();responseBody = responseEntity.getBody();responseHeaders = responseEntity.getHeaders();} finally {costMs = System.currentTimeMillis() - requestStartAt;if (reqBody.length() > 2048) {reqBody = reqBody.substring(0, 2048);}//todo log_typeMarker marker = append("log_type", "third").and(append("cost_ms", costMs)).and(append("request_url", url)).and(append("status_code", statusCode)).and(append("re_body", reqBody)).and(append("resp_body", responseBody)).and(append("req_headers", reqHeaders.toString())).and(append("reap_headers", responseHeaders != null ? responseHeaders.toString() : null));if (statusCode > 0 && statusCode < HttpStatus.BAD_REQUEST.value()) {log.info(marker, "request succeeded");} else {log.warn(marker, "request failed");}}return new HttpResp(responseHeaders, responseBody);}}
