github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

[Improvement 11072][common] Added HTTPS request support in kerberos environment (#11127) 

* [Improvement-11072][dolphinscheduler-common] Added HTTPS request support in kerberos environment

* [Improvement-11072][dolphinscheduler-common] supplement

Co-authored-by: liyangyang <liyangyang@bizseer.com>
3.1.0-release
liyangyang 2 years ago committed by GitHub
parent
7ed1e7cd74
commit
6f0a609793
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 61 additions and 37 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 24
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java
  2. 17
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java
  3. 57
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LocalServerHttpUtilsTest.java

24
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java
Unescape View File

@ -32,6 +32,7 @@ import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
@ -66,9 +67,12 @@ public class HttpUtils {
} }
private static class HttpClientInstance { private static class HttpClientInstance {
private static final CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(requestConfig).build(); private static final CloseableHttpClient httpClient = getHttpClientBuilder().build();
} }
public static HttpClientBuilder getHttpClientBuilder() {
return HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(requestConfig);
}
private static PoolingHttpClientConnectionManager cm; private static PoolingHttpClientConnectionManager cm;
@ -98,7 +102,7 @@ public class HttpUtils {
static { static {
try { try {
ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS); ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS);
ctx.init(null, new TrustManager[]{xtm}, null); ctx.init(null, new TrustManager[] {xtm}, null);
} catch (NoSuchAlgorithmException e) { } catch (NoSuchAlgorithmException e) {
logger.error("SSLContext init with NoSuchAlgorithmException", e); logger.error("SSLContext init with NoSuchAlgorithmException", e);
} catch (KeyManagementException e) { } catch (KeyManagementException e) {
@ -107,14 +111,14 @@ public class HttpUtils {
socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE); socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE);
/** set timeout、request time、socket timeout */ /** set timeout、request time、socket timeout */
requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES) requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.setExpectContinueEnabled(Boolean.TRUE) .setExpectContinueEnabled(Boolean.TRUE)
.setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST)) .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST, AuthSchemes.SPNEGO))
.setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC)) .setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC, AuthSchemes.SPNEGO))
.setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT) .setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT)
.setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true) .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true)
.build(); .build();
socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create() socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build(); .register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build();
cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry); cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
cm.setDefaultMaxPerRoute(60); cm.setDefaultMaxPerRoute(60);
cm.setMaxTotal(100); cm.setMaxTotal(100);
@ -137,7 +141,7 @@ public class HttpUtils {
/** /**
* get http response content * get http response content
* *
* @param httpget httpget * @param httpget httpget
* @param httpClient httpClient * @param httpClient httpClient
* @return http get request response content * @return http get request response content
*/ */

17
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java
Unescape View File

@ -19,14 +19,9 @@ package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.http.auth.AuthSchemeProvider;
import org.apache.http.auth.AuthScope; import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials; import org.apache.http.auth.Credentials;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpGet;
import org.apache.http.config.Lookup;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.impl.auth.SPNegoSchemeFactory;
import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClientBuilder;
@ -56,6 +51,7 @@ public class KerberosHttpClient {
private String principal; private String principal;
private String keyTabLocation; private String keyTabLocation;
public KerberosHttpClient(String principal, String keyTabLocation) { public KerberosHttpClient(String principal, String keyTabLocation) {
super(); super();
this.principal = principal; this.principal = principal;
@ -76,10 +72,7 @@ public class KerberosHttpClient {
} }
private static CloseableHttpClient buildSpengoHttpClient() { private static CloseableHttpClient buildSpengoHttpClient() {
HttpClientBuilder builder = HttpClientBuilder.create(); HttpClientBuilder builder = HttpUtils.getHttpClientBuilder();
Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
.register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true)).build();
builder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(new AuthScope(null, -1, null), new Credentials() { credentialsProvider.setCredentials(new AuthScope(null, -1, null), new Credentials() {
@Override @Override
@ -114,9 +107,9 @@ public class KerberosHttpClient {
options.put("debug", "true"); options.put("debug", "true");
return new AppConfigurationEntry[] { return new AppConfigurationEntry[] {
new AppConfigurationEntry("com.sun.security.auth.module.Krb5LoginModule", new AppConfigurationEntry("com.sun.security.auth.module.Krb5LoginModule",
AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options) }; AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options)};
} }
}; };
Set<Principal> princ = new HashSet<>(1); Set<Principal> princ = new HashSet<>(1);
princ.add(new KerberosPrincipal(userId)); princ.add(new KerberosPrincipal(userId));
Subject sub = new Subject(false, princ, new HashSet<>(), new HashSet<>()); Subject sub = new Subject(false, princ, new HashSet<>(), new HashSet<>());

57
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LocalServerHttpUtilsTest.java
Unescape View File

@ -17,25 +17,30 @@
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import com.fasterxml.jackson.databind.node.ObjectNode;
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.http.client.config.RequestConfig; import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.client.HttpClients;
import org.junit.Assert; import org.junit.Assert;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
public class LocalServerHttpUtilsTest extends TestCase{ import com.fasterxml.jackson.databind.node.ObjectNode;
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
public class LocalServerHttpUtilsTest extends TestCase {
public static final Logger logger = LoggerFactory.getLogger(LocalServerHttpUtilsTest.class); public static final Logger logger = LoggerFactory.getLogger(LocalServerHttpUtilsTest.class);
private static LocalJettyHttpServer server = null; private static LocalJettyHttpServer server = null;
public static Test suite(){
TestSuite suite=new TestSuite(); public static Test suite() {
TestSuite suite = new TestSuite();
suite.addTestSuite(LocalServerHttpUtilsTest.class); suite.addTestSuite(LocalServerHttpUtilsTest.class);
server = new LocalJettyHttpServer(suite); server = new LocalJettyHttpServer(suite);
return server; return server;
@ -44,21 +49,21 @@ public class LocalServerHttpUtilsTest extends TestCase{
public void testGetTest() throws Exception { public void testGetTest() throws Exception {
// success // success
String result = null; String result = null;
result = HttpUtils.get("http://localhost:" + server.getServerPort()+ "/test.json"); result = HttpUtils.get("http://localhost:" + server.getServerPort() + "/test.json");
Assert.assertNotNull(result); Assert.assertNotNull(result);
ObjectNode jsonObject = JSONUtils.parseObject(result); ObjectNode jsonObject = JSONUtils.parseObject(result);
Assert.assertEquals("Github",jsonObject.path("name").asText()); Assert.assertEquals("Github", jsonObject.path("name").asText());
result = HttpUtils.get("http://123.333.111.33/ccc"); result = HttpUtils.get("http://123.333.111.33/ccc");
Assert.assertNull(result); Assert.assertNull(result);
} }
public void testGetResponseContentString() { public void testGetResponseContentString() {
CloseableHttpClient httpclient = HttpClients.createDefault(); CloseableHttpClient httpclient = HttpClients.createDefault();
HttpGet httpget = new HttpGet("http://localhost:" +server.getServerPort()+"/test.json"); HttpGet httpget = new HttpGet("http://localhost:" + server.getServerPort() + "/test.json");
/** set timeout、request time、socket timeout */ /** set timeout、request time、socket timeout */
RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT) RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT)
.setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT) .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT)
.setSocketTimeout(Constants.SOCKET_TIMEOUT).setRedirectsEnabled(true).build(); .setSocketTimeout(Constants.SOCKET_TIMEOUT).setRedirectsEnabled(true).build();
httpget.setConfig(requestConfig); httpget.setConfig(requestConfig);
String responseContent = null; String responseContent = null;
@ -77,4 +82,26 @@ public class LocalServerHttpUtilsTest extends TestCase{
CloseableHttpClient httpClient2 = HttpUtils.getInstance(); CloseableHttpClient httpClient2 = HttpUtils.getInstance();
Assert.assertEquals(httpClient1, httpClient2); Assert.assertEquals(httpClient1, httpClient2);
} }
public void testKerberosHttpsGet() {
logger.info(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME));
logger.info(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH));
logger.info(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH));
String url = "https://www.apache.org/";
logger.info(KerberosHttpClient.get(url));
Assert.assertTrue(true);
}
public void testHttpsGet() {
String url = "https://www.apache.org/";
logger.info(HttpUtils.get(url));
Assert.assertTrue(true);
}
public void testHttpGet() {
String url = "http://www.apache.org/";
logger.info(HttpUtils.get(url));
Assert.assertTrue(true);
}
} }

Write Preview
Loading…
Cancel
Save