From e026d303eab424a61f9e11ab7e8a306ad027092d Mon Sep 17 00:00:00 2001 From: "felix.wang" <59079269+felix-thinkingdata@users.noreply.github.com> Date: Wed, 29 Jul 2020 19:03:44 +0800 Subject: [PATCH] Optimize the httpclient (#3337) --- .../common/utils/HttpUtils.java | 103 +++++++++++++++--- .../common/utils/HttpUtilsTest.java | 8 ++ 2 files changed, 94 insertions(+), 17 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java index 98d9cf16ec..246aba1ef0 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java @@ -18,41 +18,116 @@ package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.Constants; import org.apache.http.HttpEntity; +import org.apache.http.client.config.AuthSchemes; +import org.apache.http.client.config.CookieSpecs; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; import java.io.IOException; - +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.X509Certificate; +import java.util.Arrays; /** * http utils */ public class HttpUtils { - - + + public static final Logger logger = LoggerFactory.getLogger(HttpUtils.class); + + private static CloseableHttpClient httpClient; + + private HttpUtils() { + + } + + private static PoolingHttpClientConnectionManager cm; + + private static SSLContext ctx = null; + + private static SSLConnectionSocketFactory socketFactory; + + private static RequestConfig requestConfig; + + private static Registry socketFactoryRegistry; + + private static X509TrustManager xtm = new X509TrustManager() { + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType) { + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType) { + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return null; + } + }; + + static { + try { + ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS); + ctx.init(null, new TrustManager[] { xtm }, null); + } catch (NoSuchAlgorithmException e) { + logger.error("SSLContext init with NoSuchAlgorithmException", e); + } catch (KeyManagementException e) { + logger.error("SSLContext init with KeyManagementException", e); + } + socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE); + /** set timeout、request time、socket timeout */ + requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES) + .setExpectContinueEnabled(Boolean.TRUE) + .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST)) + .setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC)) + .setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT) + .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true) + .build(); + socketFactoryRegistry = RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build(); + cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + cm.setDefaultMaxPerRoute(60); + cm.setMaxTotal(100); + + } + + + public static synchronized CloseableHttpClient getHttpClient() { + if (null == httpClient) { + httpClient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(requestConfig).build(); + ; + } + return httpClient; + } /** * get http request content * @param url url * @return http get request response content */ public static String get(String url){ - CloseableHttpClient httpclient = HttpClients.createDefault(); + CloseableHttpClient httpclient = HttpUtils.getHttpClient(); HttpGet httpget = new HttpGet(url); - /** set timeout、request time、socket timeout */ - RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT) - .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT) - .setSocketTimeout(Constants.SOCKET_TIMEOUT) - .setRedirectsEnabled(true) - .build(); - httpget.setConfig(requestConfig); String responseContent = null; CloseableHttpResponse response = null; @@ -85,12 +160,6 @@ public class HttpUtils { httpget.releaseConnection(); httpget.abort(); } - - try { - httpclient.close(); - } catch (IOException e) { - logger.error(e.getMessage(),e); - } } return responseContent; } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java index 90ed11aed5..521260aded 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.common.utils; import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.http.impl.client.CloseableHttpClient; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -42,4 +43,11 @@ public class HttpUtilsTest { result = HttpUtils.get("https://123.333.111.33/ccc"); Assert.assertNull(result); } + + @Test + public void testGetHttpClient() { + CloseableHttpClient httpClient1 = HttpUtils.getHttpClient(); + CloseableHttpClient httpClient2 = HttpUtils.getHttpClient(); + Assert.assertEquals(httpClient1, httpClient2); + } }