Optimize the httpclient (#3337)

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java

@@ -18,17 +18,33 @@ package org.apache.dolphinscheduler.common.utils;
 
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.http.HttpEntity;
+import org.apache.http.client.config.AuthSchemes;
+import org.apache.http.client.config.CookieSpecs;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
 import java.io.IOException;
-
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.X509Certificate;
+import java.util.Arrays;
 /**
  * http utils
  */
@@ -37,22 +53,81 @@ public class HttpUtils {
 
 	public static final Logger logger = LoggerFactory.getLogger(HttpUtils.class);
 
+
+	private static CloseableHttpClient httpClient;
+
+	private HttpUtils() {
+
+	}
+
+	private static PoolingHttpClientConnectionManager cm;
+
+	private static SSLContext ctx = null;
+
+	private static SSLConnectionSocketFactory socketFactory;
+
+	private static RequestConfig requestConfig;
+
+	private static Registry<ConnectionSocketFactory> socketFactoryRegistry;
+
+	private static X509TrustManager xtm = new X509TrustManager() {
+		@Override
+		public void checkClientTrusted(X509Certificate[] chain, String authType) {
+		}
+
+		@Override
+		public void checkServerTrusted(X509Certificate[] chain, String authType) {
+		}
+
+		@Override
+		public X509Certificate[] getAcceptedIssuers() {
+			return null;
+		}
+	};
+
+	static {
+		try {
+			ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS);
+			ctx.init(null, new TrustManager[] { xtm }, null);
+		} catch (NoSuchAlgorithmException e) {
+			logger.error("SSLContext init with NoSuchAlgorithmException", e);
+		} catch (KeyManagementException e) {
+			logger.error("SSLContext init with KeyManagementException", e);
+		}
+		socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE);
+		/** set timeout、request time、socket timeout */
+		requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES)
+				.setExpectContinueEnabled(Boolean.TRUE)
+				.setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST))
+				.setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC))
+				.setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT)
+				.setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true)
+				.build();
+		socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
+				.register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build();
+		cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+		cm.setDefaultMaxPerRoute(60);
+		cm.setMaxTotal(100);
+
+	}
+
+
+	public static synchronized CloseableHttpClient getHttpClient() {
+		if (null == httpClient) {
+			httpClient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(requestConfig).build();
+			;
+		}
+		return httpClient;
+	}
 	/**
 	 * get http request content
 	 * @param url url
 	 * @return http get request response content
 	 */
 	public static String get(String url){
-		CloseableHttpClient httpclient = HttpClients.createDefault();
+		CloseableHttpClient httpclient = HttpUtils.getHttpClient();
 
 		HttpGet httpget = new HttpGet(url);
-		/** set timeout、request time、socket timeout */
-		RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT)
-				.setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT)
-				.setSocketTimeout(Constants.SOCKET_TIMEOUT)
-				.setRedirectsEnabled(true)
-				.build();
-		httpget.setConfig(requestConfig);
 		String responseContent = null;
 		CloseableHttpResponse response = null;
 
@@ -85,12 +160,6 @@ public class HttpUtils {
 				httpget.releaseConnection();
 				httpget.abort();
 			}
-
-			try {
-				httpclient.close();
-			} catch (IOException e) {
-				logger.error(e.getMessage(),e);
-			}
 		}
 		return responseContent;
 	}

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java

@@ -17,6 +17,7 @@
 package org.apache.dolphinscheduler.common.utils;
 
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.http.impl.client.CloseableHttpClient;
 import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -42,4 +43,11 @@ public class HttpUtilsTest {
 		result = HttpUtils.get("https://123.333.111.33/ccc");
 		Assert.assertNull(result);
 	}
+
+	@Test
+	public void testGetHttpClient() {
+		CloseableHttpClient httpClient1 = HttpUtils.getHttpClient();
+		CloseableHttpClient httpClient2 = HttpUtils.getHttpClient();
+		Assert.assertEquals(httpClient1, httpClient2);
+	}
 }