引入依赖
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.13</version>
</dependency>
参数配置
连接池参数:
配置 | 简介 | 默认值 |
---|---|---|
maxConnTotal | 最大连接数 | 20 |
maxConnPerRoute | 分配给同一个route(路由)最大的并发数。 举例来说,我们使用HttpClient的实现来分别请求 www.baidu.com 的资源和 www.bing.com 的资源那么他就会产生两个route |
2 |
connTimeToLive | 连接池中connection的存活时间 | 永久有效 |
maxIdleTime | 最大空闲时间 | 如果没有指定maxIdleTime的话,但是有设置evictExpiredConnections的话,默认是10秒 |
RequestConfig常用参数:
重要参数 | 默认值 | 特殊值(0) | ||
---|---|---|---|---|
超时参数 |
socketTimeout | 指客户端从服务器读取数据的timeout,超出后会抛出SocketTimeOutException | -1,未定义使用默认值 | 无限超时 |
connectTimeout | 客户端和服务器建立连接的timeout,就是http请求的三个阶段,一:建立连接;二:数据传送;三,断开连接。超时后会ConnectionTimeOutException | -1,未定义使用默认值 | 无限超时 | |
connectionRequestTimeout | 从连接池获取连接的timeout。 HttpClient中的要用连接时尝试从连接池中获取,若是在等待了一定的时间后还没有获取到可用连接(比如连接池中没有空闲连接了)则会抛出获取连接超时异常。 |
-1,未定义使用默认值 | 无限超时 |
高并发下性能优化-http连接池
首先,明确两点:
1.http连接池不是万能的,过多的长连接会占用服务器资源,导致其他服务受阻
2.http连接池只适用于请求是经常访问同一主机(或同一个接口)的情况下
3.并发数不高的情况下资源利用率低下
使用http连接池的优点:
1.复用http连接,省去了tcp的3次握手和4次挥手的时间,极大降低请求响应的时间
2.自动管理tcp连接,不用人为地释放/创建连接
使用http连接池的大致流程 :
1.创建PoolingHttpClientConnectionManager实例
2.给manager设置参数
3.给manager设置重试策略
4.给manager设置连接管理策略
5.开启监控线程,及时关闭被服务器单向断开的连接
6.构建HttpClient实例
7.创建HttpPost/HttpGet实例,并设置参数
8.获取响应,做适当的处理
9.将用完的连接放回连接池
使用方式:
java原生使用方式:
package com.powerX.httpClient;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
public class HttpClient {
public static String doGet(String httpurl) {
HttpURLConnection connection = null;
InputStream is = null;
BufferedReader br = null;
String result = null;// 返回结果字符串
try {
// 创建远程url连接对象
URL url = new URL(httpurl);
// 通过远程url连接对象打开一个连接,强转成httpURLConnection类
connection = (HttpURLConnection) url.openConnection();
// 设置连接方式:get
connection.setRequestMethod("GET");
// 设置连接主机服务器的超时时间:15000毫秒
connection.setConnectTimeout(15000);
// 设置读取远程返回的数据时间:60000毫秒
connection.setReadTimeout(60000);
// 发送请求
connection.connect();
// 通过connection连接,获取输入流
if (connection.getResponseCode() == 200) {
is = connection.getInputStream();
// 封装输入流is,并指定字符集
br = new BufferedReader(new InputStreamReader(is, "UTF-8"));
// 存放数据
StringBuffer sbf = new StringBuffer();
String temp = null;
while ((temp = br.readLine()) != null) {
sbf.append(temp);
sbf.append("\r\n");
}
result = sbf.toString();
}
} catch (MalformedURLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
// 关闭资源
if (null != br) {
try {
br.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (null != is) {
try {
is.close();
} catch (IOException e) {
e.printStackTrace();
}
}
connection.disconnect();// 关闭远程连接
}
return result;
}
public static String doPost(String httpUrl, String param) {
HttpURLConnection connection = null;
InputStream is = null;
OutputStream os = null;
BufferedReader br = null;
String result = null;
try {
URL url = new URL(httpUrl);
// 通过远程url连接对象打开连接
connection = (HttpURLConnection) url.openConnection();
// 设置连接请求方式
connection.setRequestMethod("POST");
// 设置连接主机服务器超时时间:15000毫秒
connection.setConnectTimeout(15000);
// 设置读取主机服务器返回数据超时时间:60000毫秒
connection.setReadTimeout(60000);
// 默认值为:false,当向远程服务器传送数据/写数据时,需要设置为true
connection.setDoOutput(true);
// 默认值为:true,当前向远程服务读取数据时,设置为true,该参数可有可无
connection.setDoInput(true);
// 设置传入参数的格式:请求参数应该是 name1=value1&name2=value2 的形式。
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
// 设置鉴权信息:Authorization: Bearer da3efcbf-0845-4fe3-8aba-ee040be542c0
connection.setRequestProperty("Authorization", "Bearer da3efcbf-0845-4fe3-8aba-ee040be542c0");
// 通过连接对象获取一个输出流
os = connection.getOutputStream();
// 通过输出流对象将参数写出去/传输出去,它是通过字节数组写出的
os.write(param.getBytes());
// 通过连接对象获取一个输入流,向远程读取
if (connection.getResponseCode() == 200) {
is = connection.getInputStream();
// 对输入流对象进行包装:charset根据工作项目组的要求来设置
br = new BufferedReader(new InputStreamReader(is, "UTF-8"));
StringBuffer sbf = new StringBuffer();
String temp = null;
// 循环遍历一行一行读取数据
while ((temp = br.readLine()) != null) {
sbf.append(temp);
sbf.append("\r\n");
}
result = sbf.toString();
}
} catch (MalformedURLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
// 关闭资源
if (null != br) {
try {
br.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (null != os) {
try {
os.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (null != is) {
try {
is.close();
} catch (IOException e) {
e.printStackTrace();
}
}
// 断开与远程地址url的连接
connection.disconnect();
}
return result;
}
}
apache httpClient4.5使用方式:
package com.powerX.httpClient;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
public class HttpClient4 {
public static String doGet(String url) {
CloseableHttpClient httpClient = null;
CloseableHttpResponse response = null;
String result = "";
try {
// 通过址默认配置创建一个httpClient实例
httpClient = HttpClients.createDefault();
// 创建httpGet远程连接实例
HttpGet httpGet = new HttpGet(url);
// 设置请求头信息,鉴权
httpGet.setHeader("Authorization", "Bearer da3efcbf-0845-4fe3-8aba-ee040be542c0");
// 设置配置请求参数
RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(35000)// 连接主机服务超时时间
.setConnectionRequestTimeout(35000)// 请求超时时间
.setSocketTimeout(60000)// 数据读取超时时间
.build();
// 为httpGet实例设置配置
httpGet.setConfig(requestConfig);
// 执行get请求得到返回对象
response = httpClient.execute(httpGet);
// 通过返回对象获取返回数据
HttpEntity entity = response.getEntity();
// 通过EntityUtils中的toString方法将结果转换为字符串
result = EntityUtils.toString(entity);
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
// 关闭资源
if (null != response) {
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (null != httpClient) {
try {
httpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return result;
}
public static String doPost(String url, Map<String, Object> paramMap) {
CloseableHttpClient httpClient = null;
CloseableHttpResponse httpResponse = null;
String result = "";
// 创建httpClient实例
httpClient = HttpClients.createDefault();
// 创建httpPost远程连接实例
HttpPost httpPost = new HttpPost(url);
// 配置请求参数实例
RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(35000)// 设置连接主机服务超时时间
.setConnectionRequestTimeout(35000)// 设置连接请求超时时间
.setSocketTimeout(60000)// 设置读取数据连接超时时间
.build();
// 为httpPost实例设置配置
httpPost.setConfig(requestConfig);
// 设置请求头
httpPost.addHeader("Content-Type", "application/x-www-form-urlencoded");
// 封装post请求参数
if (null != paramMap && paramMap.size() > 0) {
List<NameValuePair> nvps = new ArrayList<NameValuePair>();
// 通过map集成entrySet方法获取entity
Set<Entry<String, Object>> entrySet = paramMap.entrySet();
// 循环遍历,获取迭代器
Iterator<Entry<String, Object>> iterator = entrySet.iterator();
while (iterator.hasNext()) {
Entry<String, Object> mapEntry = iterator.next();
nvps.add(new BasicNameValuePair(mapEntry.getKey(), mapEntry.getValue().toString()));
}
// 为httpPost设置封装好的请求参数
try {
httpPost.setEntity(new UrlEncodedFormEntity(nvps, "UTF-8"));
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
}
try {
// httpClient对象执行post请求,并返回响应参数对象
httpResponse = httpClient.execute(httpPost);
// 从响应对象中获取响应内容
HttpEntity entity = httpResponse.getEntity();
result = EntityUtils.toString(entity);
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
// 关闭资源
if (null != httpResponse) {
try {
httpResponse.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (null != httpClient) {
try {
httpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return result;
}
}
springboot使用方式:
连接池方式:
统一配置参数、设置默认值:
import lombok.Data;
import lombok.experimental.Accessors;
import org.apache.commons.lang3.ObjectUtils;
import java.io.Serializable;
/**
* @author yaol
*/
@Data
@Accessors(chain = true)
public class HttpClientProperties implements Serializable {
/**
* 唯一标识, 日志辅助使用
*/
private String flag;
/**
* 代理ip, null 默认不使用代理
*/
private String httpProxyHost;
/**
* 代理端口,null 默认不使用代理
*/
private Integer httpProxyPort;
/**
* 从连接池获取连接的timeout。 / millisecond
*/
private Integer connectionRequestTimeout;
/**
* 客户端和服务器建立连接的timeout / millisecond
*/
private Integer connectTimeout;
/**
* 客户端从服务器读取数据的timeout / millisecond
*/
private Integer readTimeout;
/**
* 最大空闲连接时间 / millisecond
*/
private Integer maxIdleTime;
/**
* 服务最大连接数
*/
private Integer maxConnTotal;
/**
* 分配给同一个route(路由)最大的并发数
*/
private Integer maxConnPerRoute;
/**
* 是否需要启用证书验证
*/
private Boolean needSslValidation;
/**
* ssl签名证书路径
*/
private String sslCertPath;
/**
* ssl签名证书密码
*/
private String sslCertPassword;
/**
* 异常重试次数(null or <0: 不会重实)
*/
private Integer retryTimes;
/**
* 异常重试间隔(null or <=0: 直接重试,不做间隔) / millisecond
*/
private Integer retryIntervalTime;
/**
* 默认值设置
*/
public void buildDefault() {
this.connectTimeout = getConnectTimeout();
this.readTimeout = getReadTimeout();
this.connectionRequestTimeout = getConnectionRequestTimeout();
this.maxIdleTime = getMaxIdleTime();
this.maxConnTotal = getMaxConnTotal();
this.maxConnPerRoute = getMaxConnPerRoute();
this.needSslValidation = getNeedSslValidation();
this.retryTimes = getRetryTimes();
this.retryIntervalTime = getRetryIntervalTime();
}
public Integer getConnectTimeout() {
return ObjectUtils.defaultIfNull(this.connectTimeout, 10 * 1000);
}
public Integer getReadTimeout() {
return ObjectUtils.defaultIfNull(this.readTimeout, 10 * 1000);
}
public Integer getConnectionRequestTimeout() {
return ObjectUtils.defaultIfNull(this.connectionRequestTimeout, 3 * 1000);
}
public Integer getMaxIdleTime() {
return ObjectUtils.defaultIfNull(this.maxIdleTime, 20 * 1000);
}
public Integer getMaxConnTotal() {
return ObjectUtils.defaultIfNull(this.maxConnTotal, 200);
}
public Integer getMaxConnPerRoute() {
return ObjectUtils.defaultIfNull(this.maxConnPerRoute, 100);
}
public Boolean getNeedSslValidation() {
return ObjectUtils.defaultIfNull(this.needSslValidation, false);
}
public Integer getRetryTimes() {
return ObjectUtils.defaultIfNull(this.retryTimes, 0);
}
public Integer getRetryIntervalTime() {
return ObjectUtils.defaultIfNull(this.retryIntervalTime, 100);
}
}
ClientHttpRequestFactory工厂类工具类:
import com.alibaba.fastjson.JSON;
import com.pingxx.common.consts.LogConsts;
import com.pingxx.common.encryption.CertUtil;
import com.pingxx.common.enums.LogTypeEnum;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpHost;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.http.ssl.SSLContexts;
import org.apache.http.ssl.TrustStrategy;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import javax.net.ssl.SSLContext;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
import java.util.concurrent.TimeUnit;
import static net.logstash.logback.marker.Markers.append;
/**
* 统一ClientHttpRequestFactory
* @author yaol
*/
@Slf4j
public class ClientHttpRequestFactoryUtil {
/**
* 统一ClientHttpRequestFactory
* @param httpClientProperties
* @return
*/
public static ClientHttpRequestFactory requestFactory(HttpClientProperties httpClientProperties) {
log.info("{} httpClientProperties: {}", httpClientProperties.getFlag(), JSON.toJSONString(httpClientProperties));
try {
HttpClientBuilder httpClientBuilder = HttpClients.custom();
if (Boolean.TRUE.equals(httpClientProperties.getNeedSslValidation()) &&
StringUtils.isNotBlank(httpClientProperties.getSslCertPath())
) {
//设置证书
KeyStore keyStore = CertUtil.getKeyStore(httpClientProperties.getSslCertPath(),
httpClientProperties.getSslCertPassword(), CertUtil.KEY_STORE_TYPE_PKCS12);
if (keyStore != null) {
SSLContextBuilder sslContextBuilder = SSLContexts.custom();
try {
sslContextBuilder.loadKeyMaterial(keyStore, httpClientProperties.getSslCertPassword().toCharArray())
.loadTrustMaterial(keyStore, (chain, authType) -> true);
SSLContext sslContext = sslContextBuilder
.loadTrustMaterial(keyStore, (chain, authType) -> true)
.build();
SSLConnectionSocketFactory connectionSocketFactory = new SSLConnectionSocketFactory(
sslContext, NoopHostnameVerifier.INSTANCE);
httpClientBuilder.setSSLSocketFactory(connectionSocketFactory);
} catch (GeneralSecurityException e) {
log.error(append(LogConsts.LOG_TYPE, LogTypeEnum.EXCEPTION),
"init SSLContext error:{}", httpClientProperties.getFlag(), e);
throw new RuntimeException("init SSLContext error");
}
} else {
log.error("HttpClient need ssl cert, but keyStore is null:{}", httpClientProperties.getFlag());
throw new RuntimeException("HttpClient need ssl cert, but keyStore is null");
}
} else {
//证书绕过
TrustStrategy acceptingTrustStrategy = (x509Certificates, authType) -> true;
SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, acceptingTrustStrategy).build();
SSLConnectionSocketFactory connectionSocketFactory = new SSLConnectionSocketFactory(sslContext,
new NoopHostnameVerifier());
httpClientBuilder.setSSLSocketFactory(connectionSocketFactory);
}
//代理
if (StringUtils.isNotBlank(httpClientProperties.getHttpProxyHost()) && httpClientProperties.getHttpProxyPort() != null) {
httpClientBuilder.setProxy(new HttpHost(httpClientProperties.getHttpProxyHost(), httpClientProperties.getHttpProxyPort(), "http"));
}
// 只有io异常才会触发重试
HttpRequestRetryHandler handler = new HttpRequestRetryHandler() {
@Override
public boolean retryRequest(IOException exception, int curRetryCount, HttpContext context) {
// curRetryCount 每一次都会递增,从1开始
if (curRetryCount > retryTimes) {
return false;
}
try {
//重试延迟
if (retryTimes > 0 && retryIntervalTime > 0 ) {
Thread.sleep(curRetryCount * retryIntervalTime);
}
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
//重试场景
if (exception instanceof ConnectTimeoutException
|| exception instanceof NoHttpResponseException
|| exception instanceof ConnectException
// || exception instanceof UnknownHostException
) {
return true;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
org.apache.http.HttpRequest request = clientContext.getRequest();
boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
if (idempotent) {
// 如果请求被认为是幂等的,那么就重试。即重复执行不影响程序其他效果的
return true;
}
return false;
}
};
//重试handler
httpClientBuilder.setRetryHandler(handler);
//最大连接数
httpClientBuilder.setMaxConnTotal(httpClientProperties.getMaxConnTotal());
//分配给同一个route(路由)最大的并发数
httpClientBuilder.setMaxConnPerRoute(httpClientProperties.getMaxConnPerRoute());
//最大空闲连接时间
httpClientBuilder.evictExpiredConnections()
.evictIdleConnections(httpClientProperties.getMaxIdleTime(), TimeUnit.MILLISECONDS);
HttpComponentsClientHttpRequestFactory requestFactory = new HttpComponentsClientHttpRequestFactory();
//从连接池获取连接的timeout。
requestFactory.setConnectionRequestTimeout(httpClientProperties.getConnectionRequestTimeout());
//客户端和服务器建立连接的timeout
requestFactory.setConnectTimeout(httpClientProperties.getConnectTimeout());
//指客户端从服务器读取数据的timeout
requestFactory.setReadTimeout(httpClientProperties.getReadTimeout());
CloseableHttpClient httpClient = httpClientBuilder.build();
requestFactory.setHttpClient(httpClient);
return requestFactory;
} catch (Exception e) {
log.error("new ClientHttpRequestFactory exception: {}", httpClientProperties.getFlag(), e);
throw new RuntimeException("new ClientHttpRequestFactory exception", e);
}
}
}
获取nacos配置、初始化restTemplate:
import com.pingxx.common.util.http.ClientHttpRequestFactoryUtil;
import com.pingxx.common.util.http.HttpClientProperties;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.boot.web.client.RestTemplateBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate;
@Getter
@Setter
@Slf4j
@EnableConfigurationProperties
@Configuration
@ConfigurationProperties(prefix = "http-client")
public class HttpClientConfig {
private String httpProxyHost;
private Integer httpProxyPort;
private Integer connectionRequestTimeout;
private Integer connectTimeout;
private Integer readTimeout;
private Integer maxIdleTime;
private Integer maxConnTotal;
private Integer maxConnPerRoute;
@Bean
public ClientHttpRequestFactory requestFactory() {
HttpClientProperties httpClientProperties = new HttpClientProperties()
.setHttpProxyHost(httpProxyHost)
.setHttpProxyPort(httpProxyPort)
.setConnectionRequestTimeout(connectionRequestTimeout)
.setConnectTimeout(connectTimeout)
.setReadTimeout(readTimeout)
.setMaxIdleTime(maxIdleTime)
.setMaxConnTotal(maxConnTotal)
.setMaxConnPerRoute(maxConnPerRoute)
.setFlag("common");
httpClientProperties.buildDefault();
return ClientHttpRequestFactoryUtil.requestFactory(httpClientProperties);
}
@Bean
public RestTemplate restTemplate(RestTemplateBuilder builder) {
return builder
.requestFactory(this::requestFactory)
.build();
}
}
发起请求工具类:
package com.example.boothttp.http;
import lombok.extern.slf4j.Slf4j;
import org.slf4j.Marker;
import org.springframework.http.*;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;
import static net.logstash.logback.marker.Markers.append;
/**
* @author yaol
*/
@Slf4j
@Component
public class HttpRequestHelper {
public HttpResp send(HttpReq reqArgs, RestTemplate restTemplate) {
String url = reqArgs.getUrl();
HttpHeaders reqHeaders = reqArgs.getReqHeaders();
String reqBody = reqArgs.getReqBody();
if (reqArgs.getReqHeaders() == null) {
reqHeaders = new HttpHeaders();
}
//设置默认请求头
reqHeaders.setContentType(MediaType.APPLICATION_JSON);
reqHeaders.add(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE);
HttpEntity<String> requestEntity = new HttpEntity<>(reqBody, reqHeaders);
long requestStartAt = System.currentTimeMillis();
String responseBody = "";
int statusCode = 0;
HttpHeaders responseHeaders = null;
long costMs;
try {
ResponseEntity<String> responseEntity = restTemplate.postForEntity(url, requestEntity, String.class);
statusCode = responseEntity.getStatusCodeValue();
responseBody = responseEntity.getBody();
responseHeaders = responseEntity.getHeaders();
} finally {
costMs = System.currentTimeMillis() - requestStartAt;
if (reqBody.length() > 2048) {
reqBody = reqBody.substring(0, 2048);
}
//todo log_type
Marker marker = append("log_type", "third")
.and(append("cost_ms", costMs))
.and(append("request_url", url))
.and(append("status_code", statusCode))
.and(append("re_body", reqBody))
.and(append("resp_body", responseBody))
.and(append("req_headers", reqHeaders.toString()))
.and(append("reap_headers", responseHeaders != null ? responseHeaders.toString() : null));
if (statusCode > 0 && statusCode < HttpStatus.BAD_REQUEST.value()) {
log.info(marker, "request succeeded");
} else {
log.warn(marker, "request failed");
}
}
return new HttpResp(responseHeaders, responseBody);
}
}