From e68e994d46a3cc246375aa7b4d262a29161de354 Mon Sep 17 00:00:00 2001 From: zhuangchong <37063904+zhuangchong@users.noreply.github.com> Date: Mon, 8 Mar 2021 08:54:03 +0800 Subject: [PATCH 1/2] when Kerberos authentication is enabled,cannot get application status. (#4980) --- .../common/utils/HadoopUtils.java | 12 +- .../common/utils/KerberosHttpClient.java | 169 +++++++++--------- 2 files changed, 89 insertions(+), 92 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index d8484c2a7a..ef89a30e59 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -428,12 +428,7 @@ public class HadoopUtils implements Closeable { String applicationUrl = getApplicationUrl(applicationId); logger.info("applicationUrl={}", applicationUrl); - String responseContent; - if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)) { - responseContent = KerberosHttpClient.get(applicationUrl); - } else { - responseContent = HttpUtils.get(applicationUrl); - } + String responseContent = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false) ? KerberosHttpClient.get(applicationUrl) : HttpUtils.get(applicationUrl); if (responseContent != null) { ObjectNode jsonObject = JSONUtils.parseObject(responseContent); if (!jsonObject.has("app")) { @@ -445,7 +440,8 @@ public class HadoopUtils implements Closeable { //may be in job history String jobHistoryUrl = getJobHistoryUrl(applicationId); logger.info("jobHistoryUrl={}", jobHistoryUrl); - responseContent = HttpUtils.get(jobHistoryUrl); + responseContent = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false) ? KerberosHttpClient.get(jobHistoryUrl) : HttpUtils.get(jobHistoryUrl); + if (null != responseContent) { ObjectNode jsonObject = JSONUtils.parseObject(responseContent); if (!jsonObject.has("job")) { @@ -682,7 +678,7 @@ public class HadoopUtils implements Closeable { */ public static String getRMState(String url) { - String retStr = HttpUtils.get(url); + String retStr = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false) ? KerberosHttpClient.get(url) : HttpUtils.get(url); if (StringUtils.isEmpty(retStr)) { return null; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java index 5c1fd41900..d1977ed3f1 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java @@ -14,9 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.Constants; + import org.apache.http.auth.AuthSchemeProvider; import org.apache.http.auth.AuthScope; import org.apache.http.auth.Credentials; @@ -28,15 +30,7 @@ import org.apache.http.impl.auth.SPNegoSchemeFactory; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import javax.security.auth.Subject; -import javax.security.auth.kerberos.KerberosPrincipal; -import javax.security.auth.login.AppConfigurationEntry; -import javax.security.auth.login.Configuration; -import javax.security.auth.login.LoginContext; -import javax.security.auth.login.LoginException; import java.security.Principal; import java.security.PrivilegedAction; import java.util.HashMap; @@ -44,95 +38,103 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import javax.security.auth.Subject; +import javax.security.auth.kerberos.KerberosPrincipal; +import javax.security.auth.login.AppConfigurationEntry; +import javax.security.auth.login.Configuration; +import javax.security.auth.login.LoginContext; +import javax.security.auth.login.LoginException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * kerberos http client */ public class KerberosHttpClient { - public static final Logger logger = LoggerFactory.getLogger(KerberosHttpClient.class); private String principal; private String keyTabLocation; - public KerberosHttpClient(String principal, String keyTabLocation) { - super(); - this.principal = principal; - this.keyTabLocation = keyTabLocation; + super(); + this.principal = principal; + this.keyTabLocation = keyTabLocation; } public KerberosHttpClient(String principal, String keyTabLocation, boolean isDebug) { - this(principal, keyTabLocation); - if (isDebug) { - System.setProperty("sun.security.spnego.debug", "true"); - System.setProperty("sun.security.krb5.debug", "true"); - } + this(principal, keyTabLocation); + if (isDebug) { + System.setProperty("sun.security.spnego.debug", "true"); + System.setProperty("sun.security.krb5.debug", "true"); + } } public KerberosHttpClient(String principal, String keyTabLocation, String krb5Location, boolean isDebug) { - this(principal, keyTabLocation, isDebug); - System.setProperty("java.security.krb5.conf", krb5Location); + this(principal, keyTabLocation, isDebug); + System.setProperty("java.security.krb5.conf", krb5Location); } private static CloseableHttpClient buildSpengoHttpClient() { - HttpClientBuilder builder = HttpClientBuilder.create(); - Lookup authSchemeRegistry = RegistryBuilder.create() - .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true)).build(); - builder.setDefaultAuthSchemeRegistry(authSchemeRegistry); - BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(new AuthScope(null, -1, null), new Credentials() { - @Override - public Principal getUserPrincipal() { - return null; - } - - @Override - public String getPassword() { - return null; - } - }); - builder.setDefaultCredentialsProvider(credentialsProvider); - return builder.build(); + HttpClientBuilder builder = HttpClientBuilder.create(); + Lookup authSchemeRegistry = RegistryBuilder.create() + .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true)).build(); + builder.setDefaultAuthSchemeRegistry(authSchemeRegistry); + BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(new AuthScope(null, -1, null), new Credentials() { + @Override + public Principal getUserPrincipal() { + return null; + } + + @Override + public String getPassword() { + return null; + } + }); + builder.setDefaultCredentialsProvider(credentialsProvider); + return builder.build(); } public String get(final String url, final String userId) { - logger.info("Calling KerberosHttpClient {} {} {}", this.principal, this.keyTabLocation, url); - Configuration config = new Configuration() { - @SuppressWarnings("serial") - @Override - public AppConfigurationEntry[] getAppConfigurationEntry(String name) { - Map options = new HashMap<>(9); - options.put("useTicketCache", "false"); - options.put("useKeyTab", "true"); - options.put("keyTab", keyTabLocation); - options.put("refreshKrb5Config", "true"); - options.put("principal", principal); - options.put("storeKey", "true"); - options.put("doNotPrompt", "true"); - options.put("isInitiator", "true"); - options.put("debug", "true"); - return new AppConfigurationEntry[] { - new AppConfigurationEntry("com.sun.security.auth.module.Krb5LoginModule", - AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options) }; - } - }; - Set princ = new HashSet<>(1); - princ.add(new KerberosPrincipal(userId)); - Subject sub = new Subject(false, princ, new HashSet<>(), new HashSet<>()); - - LoginContext lc; - try { - lc = new LoginContext("", sub, null, config); - lc.login(); - Subject serviceSubject = lc.getSubject(); - return Subject.doAs(serviceSubject, (PrivilegedAction) () -> { - CloseableHttpClient httpClient = buildSpengoHttpClient(); - HttpGet httpget = new HttpGet(url); - return HttpUtils.getResponseContentString(httpget, httpClient); - }); - } catch (LoginException le) { - logger.error("Kerberos authentication failed ", le); - } - return null; + logger.info("Calling KerberosHttpClient {} {} {}", this.principal, this.keyTabLocation, url); + Configuration config = new Configuration() { + @SuppressWarnings("serial") + @Override + public AppConfigurationEntry[] getAppConfigurationEntry(String name) { + Map options = new HashMap<>(9); + options.put("useTicketCache", "false"); + options.put("useKeyTab", "true"); + options.put("keyTab", keyTabLocation); + options.put("refreshKrb5Config", "true"); + options.put("principal", principal); + options.put("storeKey", "true"); + options.put("doNotPrompt", "true"); + options.put("isInitiator", "true"); + options.put("debug", "true"); + return new AppConfigurationEntry[] { + new AppConfigurationEntry("com.sun.security.auth.module.Krb5LoginModule", + AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options) }; + } + }; + Set princ = new HashSet<>(1); + princ.add(new KerberosPrincipal(userId)); + Subject sub = new Subject(false, princ, new HashSet<>(), new HashSet<>()); + + LoginContext lc; + try { + lc = new LoginContext("", sub, null, config); + lc.login(); + Subject serviceSubject = lc.getSubject(); + return Subject.doAs(serviceSubject, (PrivilegedAction) () -> { + CloseableHttpClient httpClient = buildSpengoHttpClient(); + HttpGet httpget = new HttpGet(url); + return HttpUtils.getResponseContentString(httpget, httpClient); + }); + } catch (LoginException le) { + logger.error("Kerberos authentication failed ", le); + } + return null; } /** @@ -143,14 +145,13 @@ public class KerberosHttpClient { */ public static String get(String url) { - String responseContent; - KerberosHttpClient kerberosHttpClient = new KerberosHttpClient( - PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), - PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH), - PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), true); - responseContent = kerberosHttpClient.get(url, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)); - return responseContent; + String responseContent; + KerberosHttpClient kerberosHttpClient = new KerberosHttpClient( + PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), + PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH), + PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), true); + responseContent = kerberosHttpClient.get(url, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)); + return responseContent; } - } From 71bd231c9564837afe6f9ea380b42e7f057a610a Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Mon, 8 Mar 2021 08:58:38 +0800 Subject: [PATCH 2/2] [Improvement][UI] Unify form model by m-list-box (#4982) --- .../pages/dag/_source/formModel/formModel.vue | 124 +++++------ .../tasks/_source/dependItemList.vue | 1 + .../_source/formModel/tasks/conditions.vue | 4 +- .../dag/_source/formModel/tasks/datax.vue | 18 +- .../dag/_source/formModel/tasks/dependent.vue | 10 +- .../dag/_source/formModel/tasks/flink.vue | 1 - .../dag/_source/formModel/tasks/http.vue | 19 +- .../pages/dag/_source/formModel/tasks/mr.vue | 2 +- .../dag/_source/formModel/tasks/pre_tasks.vue | 49 +++-- .../dag/_source/formModel/tasks/python.vue | 3 +- .../pages/dag/_source/formModel/tasks/sql.vue | 4 +- .../dag/_source/formModel/tasks/sqoop.vue | 200 ++++++++---------- .../_source/formModel/tasks/sub_process.vue | 44 ++-- .../dag/_source/formModel/tasks/waterdrop.vue | 2 +- .../src/sass/common/index.scss | 8 +- 15 files changed, 223 insertions(+), 266 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index 744c041e97..fe95e7af85 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -30,99 +30,83 @@
-
-
{{$t('Node name')}}
-
- + +
{{$t('Node name')}}
+
+ +
-
+ -
-
{{$t('Run flag')}}
-
- + +
{{$t('Run flag')}}
+
+ + {{$t('Normal')}} + {{$t('Prohibition execution')}} +
-
+ -
-
- {{$t('Description')}} + +
{{$t('Description')}}
+
+ +
-
- -
-
+ -
-
- {{$t('Task priority')}} -
-
+ +
{{$t('Task priority')}}
+
{{$t('Worker group')}}
-
+ -
-
- {{$t('Number of failed retries')}} -
-
+ +
{{$t('Number of failed retries')}}
+
({{$t('Times')}}) {{$t('Failed retry interval')}} ({{$t('Minute')}})
-
+ -
-
- {{$t('Delay execution time')}} -
-
+ +
{{$t('Delay execution time')}}
+
({{$t('Minute')}})
-
+ -
-
- {{$t('State')}} -
-
+ +
{{$t('State')}}
+
@@ -133,12 +117,10 @@
-
-
-
- {{$t('State')}} -
-
+ + +
{{$t('State')}}
+
@@ -149,7 +131,7 @@
-
+ { if (!this.dependItemList.length) { + if (!this.projectList.length) return let projectId = this.projectList[0].value this._getProcessByProjectId(projectId).then(definitionList => { let value = definitionList[0].value diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue index 3cfb1625be..60a6806c62 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue @@ -15,7 +15,7 @@ * limitations under the License. */