Browse Source

[Fix-3262][common] When you request the URL through applicationID to get the application status, you cannot get it if Kerberos authentication is enabled (#3264)

* fix bug #3165   get resource.storage.type value   toUpperCase

* fix bug #3176  optimize Gets the value of this property “resource.storage.type”

* fix When you request the URL through applicationID to get the application status, you cannot get it if Kerberos authentication is enabled

* fix When you request the URL through applicationID to get the application status, you cannot get it if Kerberos authentication is enabled

* fix When you request the URL through applicationID to get the application status, you cannot get it if Kerberos authentication is enabled

* fix miss

* add KerberosHttpClient fix  kerberos bug

* fix map init

* add juint test

* Extraction of public methods

* Extraction of public methods

* Fix code quality

* Fix code quality

* Fix code quality

* Fix code quality

* Fix code quality

* Fix code quality

* Fix code quality

* rebuild test

* rebuild test

* check style

* check style

* check style

* revert UT POM

* Kerberos judgment goes up to hadoopUtils

* fix merge

* Remove connection pool shutdown

Co-authored-by: dailidong <dailidong66@gmail.com>
pull/3/MERGE
felix.wang 4 years ago committed by GitHub
parent
commit
5584f0cb4d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 7
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
  2. 76
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java
  3. 156
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java
  4. 55
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java
  5. 46
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClientTest.java
  6. 1
      pom.xml

7
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java

@ -417,7 +417,12 @@ public class HadoopUtils implements Closeable {
String applicationUrl = getApplicationUrl(applicationId);
logger.info("applicationUrl={}", applicationUrl);
String responseContent = HttpUtils.get(applicationUrl);
String responseContent ;
if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)) {
responseContent = KerberosHttpClient.get(applicationUrl);
} else {
responseContent = HttpUtils.get(applicationUrl);
}
if (responseContent != null) {
ObjectNode jsonObject = JSONUtils.parseObject(responseContent);
result = jsonObject.path("app").path("finalStatus").asText();

76
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java

@ -128,40 +128,50 @@ public class HttpUtils {
CloseableHttpClient httpclient = HttpUtils.getInstance();
HttpGet httpget = new HttpGet(url);
String responseContent = null;
CloseableHttpResponse response = null;
try {
response = httpclient.execute(httpget);
//check response status is 200
if (response.getStatusLine().getStatusCode() == 200) {
HttpEntity entity = response.getEntity();
if (entity != null) {
responseContent = EntityUtils.toString(entity, Constants.UTF_8);
}else{
logger.warn("http entity is null");
}
}else{
logger.error("http get:{} response status code is not 200!", response.getStatusLine().getStatusCode());
}
}catch (Exception e){
logger.error(e.getMessage(),e);
}finally {
try {
if (response != null) {
EntityUtils.consume(response.getEntity());
response.close();
}
} catch (IOException e) {
logger.error(e.getMessage(),e);
}
if (!httpget.isAborted()) {
httpget.releaseConnection();
httpget.abort();
}
return getResponseContentString(httpget,httpclient);
}
/**
* get http response content
*
* @param httpget httpget
* @param httpClient httpClient
* @return http get request response content
*/
public static String getResponseContentString(HttpGet httpget, CloseableHttpClient httpClient) {
String responseContent = null;
CloseableHttpResponse response = null;
try {
response = httpClient.execute(httpget);
// check response status is 200
if (response.getStatusLine().getStatusCode() == 200) {
HttpEntity entity = response.getEntity();
if (entity != null) {
responseContent = EntityUtils.toString(entity, Constants.UTF_8);
} else {
logger.warn("http entity is null");
}
return responseContent;
} else {
logger.error("http get:{} response status code is not 200!", response.getStatusLine().getStatusCode());
}
} catch (IOException ioe) {
logger.error(ioe.getMessage(), ioe);
} finally {
try {
if (response != null) {
EntityUtils.consume(response.getEntity());
response.close();
}
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
if (!httpget.isAborted()) {
httpget.releaseConnection();
httpget.abort();
}
}
return responseContent;
}
}

156
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java

@ -0,0 +1,156 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.http.auth.AuthSchemeProvider;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.config.Lookup;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.impl.auth.SPNegoSchemeFactory;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.security.auth.Subject;
import javax.security.auth.kerberos.KerberosPrincipal;
import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.Configuration;
import javax.security.auth.login.LoginContext;
import javax.security.auth.login.LoginException;
import java.security.Principal;
import java.security.PrivilegedAction;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
/**
* kerberos http client
*/
public class KerberosHttpClient {
public static final Logger logger = LoggerFactory.getLogger(KerberosHttpClient.class);
private String principal;
private String keyTabLocation;
public KerberosHttpClient(String principal, String keyTabLocation) {
super();
this.principal = principal;
this.keyTabLocation = keyTabLocation;
}
public KerberosHttpClient(String principal, String keyTabLocation, boolean isDebug) {
this(principal, keyTabLocation);
if (isDebug) {
System.setProperty("sun.security.spnego.debug", "true");
System.setProperty("sun.security.krb5.debug", "true");
}
}
public KerberosHttpClient(String principal, String keyTabLocation, String krb5Location, boolean isDebug) {
this(principal, keyTabLocation, isDebug);
System.setProperty("java.security.krb5.conf", krb5Location);
}
private static CloseableHttpClient buildSpengoHttpClient() {
HttpClientBuilder builder = HttpClientBuilder.create();
Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
.register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true)).build();
builder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(new AuthScope(null, -1, null), new Credentials() {
@Override
public Principal getUserPrincipal() {
return null;
}
@Override
public String getPassword() {
return null;
}
});
builder.setDefaultCredentialsProvider(credentialsProvider);
return builder.build();
}
public String get(final String url, final String userId) {
logger.info("Calling KerberosHttpClient {} {} {}", this.principal, this.keyTabLocation, url);
Configuration config = new Configuration() {
@SuppressWarnings("serial")
@Override
public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
Map<String, Object> options = new HashMap<>(9);
options.put("useTicketCache", "false");
options.put("useKeyTab", "true");
options.put("keyTab", keyTabLocation);
options.put("refreshKrb5Config", "true");
options.put("principal", principal);
options.put("storeKey", "true");
options.put("doNotPrompt", "true");
options.put("isInitiator", "true");
options.put("debug", "true");
return new AppConfigurationEntry[] {
new AppConfigurationEntry("com.sun.security.auth.module.Krb5LoginModule",
AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options) };
}
};
Set<Principal> princ = new HashSet<>(1);
princ.add(new KerberosPrincipal(userId));
Subject sub = new Subject(false, princ, new HashSet<>(), new HashSet<>());
LoginContext lc;
try {
lc = new LoginContext("", sub, null, config);
lc.login();
Subject serviceSubject = lc.getSubject();
return Subject.doAs(serviceSubject, (PrivilegedAction<String>) () -> {
CloseableHttpClient httpClient = buildSpengoHttpClient();
HttpGet httpget = new HttpGet(url);
return HttpUtils.getResponseContentString(httpget, httpClient);
});
} catch (LoginException le) {
logger.error("Kerberos authentication failed ", le);
}
return null;
}
/**
* get http request content by kerberosClient
*
* @param url url
* @return http get request response content
*/
public static String get(String url) {
String responseContent;
KerberosHttpClient kerberosHttpClient = new KerberosHttpClient(
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH),
PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), true);
responseContent = kerberosHttpClient.get(url, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME));
return responseContent;
}
}

55
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java

@ -17,7 +17,13 @@
package org.apache.dolphinscheduler.common.utils;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@ -28,26 +34,53 @@ import org.slf4j.LoggerFactory;
*/
public class HttpUtilsTest {
public static final Logger logger = LoggerFactory.getLogger(HttpUtilsTest.class);
private HadoopUtils hadoopUtils = HadoopUtils.getInstance();
public static final Logger logger = LoggerFactory.getLogger(HttpUtilsTest.class);
@Test
public void testGetTest() {
// success
String result = HttpUtils.get("https://github.com/manifest.json");
Assert.assertNotNull(result);
ObjectNode jsonObject = JSONUtils.parseObject(result);
Assert.assertEquals("GitHub", jsonObject.path("name").asText());
result = HttpUtils.get("https://123.333.111.33/ccc");
Assert.assertNull(result);
}
@Test
public void testGetByKerberos() {
try {
String applicationUrl = hadoopUtils.getApplicationUrl("application_1542010131334_0029");
String responseContent;
responseContent = HttpUtils.get(applicationUrl);
Assert.assertNull(responseContent);
@Test
public void testGetTest(){
//success
String result = HttpUtils.get("https://github.com/manifest.json");
Assert.assertNotNull(result);
ObjectNode jsonObject = JSONUtils.parseObject(result);
Assert.assertEquals("GitHub", jsonObject.path("name").asText());
result = HttpUtils.get("https://123.333.111.33/ccc");
Assert.assertNull(result);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
@Test
public void testGetResponseContentString() {
CloseableHttpClient httpclient = HttpClients.createDefault();
HttpGet httpget = new HttpGet("https://github.com/manifest.json");
/** set timeout、request time、socket timeout */
RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT)
.setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT)
.setSocketTimeout(Constants.SOCKET_TIMEOUT).setRedirectsEnabled(true).build();
httpget.setConfig(requestConfig);
String responseContent = HttpUtils.getResponseContentString(httpget, httpclient);
Assert.assertNotNull(responseContent);
}
@Test
public void testGetHttpClient() {
CloseableHttpClient httpClient1 = HttpUtils.getInstance();
CloseableHttpClient httpClient2 = HttpUtils.getInstance();
Assert.assertEquals(httpClient1, httpClient2);
}
}

46
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClientTest.java

@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* KerberosHttpClient test
*/
public class KerberosHttpClientTest {
public static final Logger logger = LoggerFactory.getLogger(KerberosHttpClientTest.class);
private HadoopUtils hadoopUtils = HadoopUtils.getInstance();
@Test
public void get() {
try {
String applicationUrl = hadoopUtils.getApplicationUrl("application_1542010131334_0029");
String responseContent;
KerberosHttpClient kerberosHttpClient = new KerberosHttpClient(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH), PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), true);
responseContent = kerberosHttpClient.get(applicationUrl,
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME));
Assert.assertNull(responseContent);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
}

1
pom.xml

@ -779,6 +779,7 @@
<include>**/common/utils/TaskParametersUtilsTest.java</include>
<include>**/common/utils/HadoopUtilsTest.java</include>
<include>**/common/utils/HttpUtilsTest.java</include>
<include>**/common/utils/KerberosHttpClientTest.java</include>
<include>**/common/ConstantsTest.java</include>
<include>**/common/utils/HadoopUtils.java</include>
<include>**/common/utils/RetryerUtilsTest.java</include>

Loading…
Cancel
Save