Merge remote-tracking branch 'upstream/dev' into params-trans

dolphinscheduler-alert/src/main/resources/alert.properties

@@ -21,7 +21,7 @@
 #eg : Alert Server Listener port
 
 #alert.plugin.dir config the Alert Plugin dir . AlertServer while find and load the Alert Plugin Jar from this dir when deploy and start AlertServer on the server .
-#eg :alert.plugin.dir=/opt/soft/spi/lib/plugin/alert
+alert.plugin.dir=./lib/plugin/alert
 
 #maven.local.repository=/Users/gaojun/Documents/jianguoyun/localRepository
 

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java

@@ -96,10 +96,14 @@ public class DataSourceController extends BaseController {
             @ApiImplicitParam(name = "host", value = "DATA_SOURCE_HOST", required = true, dataType = "String"),
             @ApiImplicitParam(name = "port", value = "DATA_SOURCE_PORT", required = true, dataType = "String"),
             @ApiImplicitParam(name = "database", value = "DATABASE_NAME", required = true, dataType = "String"),
+            @ApiImplicitParam(name = "principal", value = "DATA_SOURCE_PRINCIPAL", dataType = "String"),
             @ApiImplicitParam(name = "userName", value = "USER_NAME", required = true, dataType = "String"),
             @ApiImplicitParam(name = "password", value = "PASSWORD", dataType = "String"),
             @ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"),
-            @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String")
+            @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String"),
+            @ApiImplicitParam(name = "javaSecurityKrb5Conf", value = "DATA_SOURCE_KERBEROS_KRB5_CONF", dataType = "String"),
+            @ApiImplicitParam(name = "loginUserKeytabUsername", value = "DATA_SOURCE_KERBEROS_KEYTAB_USERNAME", dataType = "String"),
+            @ApiImplicitParam(name = "loginUserKeytabPath", value = "DATA_SOURCE_KERBEROS_KEYTAB_PATH", dataType = "String")
     })
     @PostMapping(value = "/create")
     @ResponseStatus(HttpStatus.CREATED)
@@ -115,10 +119,14 @@ public class DataSourceController extends BaseController {
                                    @RequestParam(value = "userName") String userName,
                                    @RequestParam(value = "password") String password,
                                    @RequestParam(value = "connectType") DbConnectType connectType,
-                                   @RequestParam(value = "other") String other) {
+                                   @RequestParam(value = "other") String other,
+                                   @RequestParam(value = "javaSecurityKrb5Conf", required = false) String javaSecurityKrb5Conf,
+                                   @RequestParam(value = "loginUserKeytabUsername", required = false) String loginUserKeytabUsername,
+                                   @RequestParam(value = "loginUserKeytabPath", required = false) String loginUserKeytabPath) {
         logger.info("login user {} create datasource name: {}, note: {}, type: {}, host: {}, port: {}, database : {}, principal: {}, userName : {}, connectType: {}, other: {}",
                 loginUser.getUserName(), name, note, type, host, port, database, principal, userName, connectType, other);
-        String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other);
+        String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other,
+                javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath);
         return dataSourceService.createDataSource(loginUser, name, note, type, parameter);
     }
 
@@ -149,10 +157,14 @@ public class DataSourceController extends BaseController {
             @ApiImplicitParam(name = "host", value = "DATA_SOURCE_HOST", required = true, dataType = "String"),
             @ApiImplicitParam(name = "port", value = "DATA_SOURCE_PORT", required = true, dataType = "String"),
             @ApiImplicitParam(name = "database", value = "DATABASE_NAME", required = true, dataType = "String"),
+            @ApiImplicitParam(name = "principal", value = "DATA_SOURCE_PRINCIPAL", dataType = "String"),
             @ApiImplicitParam(name = "userName", value = "USER_NAME", required = true, dataType = "String"),
             @ApiImplicitParam(name = "password", value = "PASSWORD", dataType = "String"),
             @ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"),
-            @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String")
+            @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String"),
+            @ApiImplicitParam(name = "javaSecurityKrb5Conf", value = "DATA_SOURCE_KERBEROS_KRB5_CONF", dataType = "String"),
+            @ApiImplicitParam(name = "loginUserKeytabUsername", value = "DATA_SOURCE_KERBEROS_KEYTAB_USERNAME", dataType = "String"),
+            @ApiImplicitParam(name = "loginUserKeytabPath", value = "DATA_SOURCE_KERBEROS_KEYTAB_PATH", dataType = "String")
     })
     @PostMapping(value = "/update")
     @ResponseStatus(HttpStatus.OK)
@@ -169,10 +181,14 @@ public class DataSourceController extends BaseController {
                                    @RequestParam(value = "userName") String userName,
                                    @RequestParam(value = "password") String password,
                                    @RequestParam(value = "connectType") DbConnectType connectType,
-                                   @RequestParam(value = "other") String other) {
+                                   @RequestParam(value = "other") String other,
+                                   @RequestParam(value = "javaSecurityKrb5Conf", required = false) String javaSecurityKrb5Conf,
+                                   @RequestParam(value = "loginUserKeytabUsername", required = false) String loginUserKeytabUsername,
+                                   @RequestParam(value = "loginUserKeytabPath", required = false) String loginUserKeytabPath) {
         logger.info("login user {} updateProcessInstance datasource name: {}, note: {}, type: {}, connectType: {}, other: {}",
                 loginUser.getUserName(), name, note, type, connectType, other);
-        String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other);
+        String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other,
+                javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath);
         return dataSourceService.updateDataSource(id, loginUser, name, note, type, parameter);
     }
 
@@ -274,10 +290,14 @@ public class DataSourceController extends BaseController {
             @ApiImplicitParam(name = "host", value = "DATA_SOURCE_HOST", required = true, dataType = "String"),
             @ApiImplicitParam(name = "port", value = "DATA_SOURCE_PORT", required = true, dataType = "String"),
             @ApiImplicitParam(name = "database", value = "DATABASE_NAME", required = true, dataType = "String"),
+            @ApiImplicitParam(name = "principal", value = "DATA_SOURCE_PRINCIPAL", dataType = "String"),
             @ApiImplicitParam(name = "userName", value = "USER_NAME", required = true, dataType = "String"),
             @ApiImplicitParam(name = "password", value = "PASSWORD", dataType = "String"),
             @ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"),
-            @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String")
+            @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String"),
+            @ApiImplicitParam(name = "javaSecurityKrb5Conf", value = "DATA_SOURCE_KERBEROS_KRB5_CONF", dataType = "String"),
+            @ApiImplicitParam(name = "loginUserKeytabUsername", value = "DATA_SOURCE_KERBEROS_KEYTAB_USERNAME", dataType = "String"),
+            @ApiImplicitParam(name = "loginUserKeytabPath", value = "DATA_SOURCE_KERBEROS_KEYTAB_PATH", dataType = "String")
     })
     @PostMapping(value = "/connect")
     @ResponseStatus(HttpStatus.OK)
@@ -293,10 +313,14 @@ public class DataSourceController extends BaseController {
                                     @RequestParam(value = "userName") String userName,
                                     @RequestParam(value = "password") String password,
                                     @RequestParam(value = "connectType") DbConnectType connectType,
-                                    @RequestParam(value = "other") String other) {
+                                    @RequestParam(value = "other") String other,
+                                    @RequestParam(value = "javaSecurityKrb5Conf", required = false) String javaSecurityKrb5Conf,
+                                    @RequestParam(value = "loginUserKeytabUsername", required = false) String loginUserKeytabUsername,
+                                    @RequestParam(value = "loginUserKeytabPath", required = false) String loginUserKeytabPath) {
         logger.info("login user {}, connect datasource: {}, note: {}, type: {}, connectType: {}, other: {}",
                 loginUser.getUserName(), name, note, type, connectType, other);
-        String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other);
+        String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other,
+                javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath);
         return dataSourceService.checkConnection(type, parameter);
     }
 

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java

@@ -267,6 +267,9 @@ public class DataSourceService extends BaseService {
         map.put(HOST, host);
         map.put(PORT, port);
         map.put(PRINCIPAL, datasourceForm.getPrincipal());
+        map.put(Constants.KERBEROS_KRB5_CONF_PATH, datasourceForm.getJavaSecurityKrb5Conf());
+        map.put(Constants.KERBEROS_KEY_TAB_USERNAME, datasourceForm.getLoginUserKeytabUsername());
+        map.put(Constants.KERBEROS_KEY_TAB_PATH, datasourceForm.getLoginUserKeytabPath());
         map.put(DATABASE, database);
         map.put(USER_NAME, datasourceForm.getUser());
         map.put(OTHER, otherMap);
@@ -424,7 +427,8 @@ public class DataSourceService extends BaseService {
      */
     public String buildParameter(DbType type, String host,
                                  String port, String database, String principal, String userName,
-                                 String password, DbConnectType connectType, String other) {
+                                 String password, DbConnectType connectType, String other,
+                                 String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath) {
 
         String address = buildAddress(type, host, port, connectType);
         Map<String, Object> parameterMap = new LinkedHashMap<String, Object>(6);
@@ -467,6 +471,9 @@ public class DataSourceService extends BaseService {
         if (CommonUtils.getKerberosStartupState()
                 && (type == DbType.HIVE || type == DbType.SPARK)) {
             parameterMap.put(Constants.PRINCIPAL, principal);
+            parameterMap.put(Constants.KERBEROS_KRB5_CONF_PATH, javaSecurityKrb5Conf);
+            parameterMap.put(Constants.KERBEROS_KEY_TAB_USERNAME, loginUserKeytabUsername);
+            parameterMap.put(Constants.KERBEROS_KEY_TAB_PATH, loginUserKeytabPath);
         }
 
         Map<String, String> map = JSONUtils.toMap(other);

dolphinscheduler-api/src/main/resources/i18n/messages.properties

@@ -125,6 +125,10 @@ TENANT_CODE=os tenant code
 QUEUE_NAME=queue name
 PASSWORD=password
 DATA_SOURCE_OTHER=jdbc connection params, format:{"key1":"value1",...}
+DATA_SOURCE_PRINCIPAL=principal
+DATA_SOURCE_KERBEROS_KRB5_CONF=the kerberos authentication parameter java.security.krb5.conf
+DATA_SOURCE_KERBEROS_KEYTAB_USERNAME=the kerberos authentication parameter login.user.keytab.username
+DATA_SOURCE_KERBEROS_KEYTAB_PATH=the kerberos authentication parameter login.user.keytab.path
 PROJECT_TAG=project related operation
 CREATE_PROJECT_NOTES=create project 
 PROJECT_DESC=project description

dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties

@@ -125,6 +125,10 @@ TENANT_CODE=os tenant code
 QUEUE_NAME=queue name
 PASSWORD=password
 DATA_SOURCE_OTHER=jdbc connection params, format:{"key1":"value1",...}
+DATA_SOURCE_PRINCIPAL=principal
+DATA_SOURCE_KERBEROS_KRB5_CONF=the kerberos authentication parameter java.security.krb5.conf
+DATA_SOURCE_KERBEROS_KEYTAB_USERNAME=the kerberos authentication parameter login.user.keytab.username
+DATA_SOURCE_KERBEROS_KEYTAB_PATH=the kerberos authentication parameter login.user.keytab.path
 PROJECT_TAG=project related operation
 CREATE_PROJECT_NOTES=create project 
 PROJECT_DESC=project description

dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties

@@ -119,6 +119,10 @@ TENANT_CODE=操作系统租户
 QUEUE_NAME=队列名
 PASSWORD=密码
 DATA_SOURCE_OTHER=jdbc连接参数，格式为:{"key1":"value1",...}
+DATA_SOURCE_PRINCIPAL=principal
+DATA_SOURCE_KERBEROS_KRB5_CONF=kerberos认证参数 java.security.krb5.conf
+DATA_SOURCE_KERBEROS_KEYTAB_USERNAME=kerberos认证参数 login.user.keytab.username
+DATA_SOURCE_KERBEROS_KEYTAB_PATH=kerberos认证参数 login.user.keytab.path
 PROJECT_TAG=项目相关操作
 CREATE_PROJECT_NOTES=创建项目
 PROJECT_DESC=项目描述

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java

@@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.DbConnectType;
 import org.apache.dolphinscheduler.common.enums.DbType;
 import org.apache.dolphinscheduler.common.enums.UserType;
+import org.apache.dolphinscheduler.common.utils.CommonUtils;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
@@ -51,7 +52,7 @@ import org.powermock.modules.junit4.PowerMockRunner;
 
 @RunWith(PowerMockRunner.class)
 @PowerMockIgnore({"sun.security.*", "javax.net.*"})
-@PrepareForTest({DataSourceFactory.class})
+@PrepareForTest({DataSourceFactory.class, CommonUtils.class})
 public class DataSourceServiceTest {
 
 
@@ -68,7 +69,7 @@ public class DataSourceServiceTest {
         String dataSourceName = "dataSource01";
         String dataSourceDesc = "test dataSource";
         DbType dataSourceType = DbType.POSTGRESQL;
-        String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null);
+        String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null, null, null, null);
 
         // data source exits
         List<DataSource> dataSourceList = new ArrayList<>();
@@ -110,7 +111,7 @@ public class DataSourceServiceTest {
         String dataSourceName = "dataSource01";
         String dataSourceDesc = "test dataSource";
         DbType dataSourceType = DbType.POSTGRESQL;
-        String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null);
+        String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null, null, null,  null);
 
         // data source not exits
         PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(null);
@@ -274,24 +275,35 @@ public class DataSourceServiceTest {
     @Test
     public void buildParameter() {
         String param = dataSourceService.buildParameter(DbType.ORACLE, "192.168.9.1", "1521", "im"
-                , "", "test", "test", DbConnectType.ORACLE_SERVICE_NAME, "");
+                , "", "test", "test", DbConnectType.ORACLE_SERVICE_NAME, "", "", "","");
         String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\","
                 + "\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}";
         Assert.assertEquals(expected, param);
+
+        PowerMockito.mockStatic(CommonUtils.class);
+        PowerMockito.when(CommonUtils.getKerberosStartupState()).thenReturn(true);
+        PowerMockito.when(CommonUtils.encodePassword(Mockito.anyString())).thenReturn("test");
+        param = dataSourceService.buildParameter(DbType.HIVE, "192.168.9.1", "10000", "im"
+                , "hive/hdfs-mycluster@ESZ.COM", "test", "test", null, "", "/opt/krb5.conf", "test2/hdfs-mycluster@ESZ.COM", "/opt/hdfs.headless.keytab");
+        expected = "{\"type\":null,\"address\":\"jdbc:hive2://192.168.9.1:10000\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:hive2://192.168.9.1:10000/im;principal=hive/hdfs-mycluster@ESZ.COM\","
+                + "\"user\":\"test\",\"password\":\"test\",\"principal\":\"hive/hdfs-mycluster@ESZ.COM\",\"javaSecurityKrb5Conf\":\"/opt/krb5.conf\","
+                + "\"loginUserKeytabUsername\":\"test2/hdfs-mycluster@ESZ.COM\",\"loginUserKeytabPath\":\"/opt/hdfs.headless.keytab\"}";
+        Assert.assertEquals(expected, param);
+
     }
 
     @Test
     public void buildParameterWithDecodePassword() {
         PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true");
         String param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im"
-                , "", "test", "123456", null, "");
+                , "", "test", "123456", null, "", "", "", "");
         String expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\","
                 + "\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}";
         Assert.assertEquals(expected, param);
 
         PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false");
         param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im"
-                , "", "test", "123456", null, "");
+                , "", "test", "123456", null, "", "", "", "");
         expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"123456\"}";
         Assert.assertEquals(expected, param);
     }
@@ -316,7 +328,7 @@ public class DataSourceServiceTest {
     @Test
     public void testCheckConnection() throws Exception {
         DbType dataSourceType = DbType.POSTGRESQL;
-        String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null);
+        String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null, null, null, null);
 
         PowerMockito.mockStatic(DataSourceFactory.class);
         PowerMockito.when(DataSourceFactory.getDatasource(Mockito.any(), Mockito.anyString())).thenReturn(null);

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@@ -956,7 +956,9 @@ public final class Constants {
     public static final String PRINCIPAL = "principal";
     public static final String OTHER = "other";
     public static final String ORACLE_DB_CONNECT_TYPE = "connectType";
-
+    public static final String KERBEROS_KRB5_CONF_PATH = "javaSecurityKrb5Conf";
+    public static final String KERBEROS_KEY_TAB_USERNAME = "loginUserKeytabUsername";
+    public static final String KERBEROS_KEY_TAB_PATH = "loginUserKeytabPath";
 
     /**
      * session timeout

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java

@@ -86,13 +86,26 @@ public class CommonUtils {
      * @throws Exception errors
      */
     public static void loadKerberosConf() throws Exception {
+        loadKerberosConf(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH),
+                PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
+                PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH));
+    }
+
+    /**
+     * load kerberos configuration
+     * @param javaSecurityKrb5Conf javaSecurityKrb5Conf
+     * @param loginUserKeytabUsername loginUserKeytabUsername
+     * @param loginUserKeytabPath loginUserKeytabPath
+     * @throws Exception errors
+     */
+    public static void loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath) throws Exception {
         if (CommonUtils.getKerberosStartupState()) {
-            System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH));
+            System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, StringUtils.defaultIfBlank(javaSecurityKrb5Conf, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)));
             Configuration configuration = new Configuration();
             configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, Constants.KERBEROS);
             UserGroupInformation.setConfiguration(configuration);
-            UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
-                    PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH));
+            UserGroupInformation.loginUserFromKeytab(StringUtils.defaultIfBlank(loginUserKeytabUsername, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)),
+                    StringUtils.defaultIfBlank(loginUserKeytabPath, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)));
         }
     }
 

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java

@@ -389,6 +389,16 @@ public class OSUtils {
         return null;
     }
 
+    /**
+     *  get sudo command
+     * @param tenantCode tenantCode
+     * @param command command
+     * @return result of sudo execute command
+     */
+    public static String getSudoCmd(String tenantCode, String command) {
+        return StringUtils.isEmpty(tenantCode) ? command : "sudo -u " + tenantCode + " " + command;
+    }
+
     /**
      * Execute the corresponding command of Linux or Windows
      *

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java

@@ -62,6 +62,10 @@ public class StringUtils {
         return str == null ? null : str.trim();
     }
 
+    public static String defaultIfBlank(String str, String defaultStr) {
+        return isBlank(str) ? defaultStr : str;
+    }
+
     public static boolean equalsIgnoreCase(String str1, String str2) {
         return str1 == null ? str2 == null : str1.equalsIgnoreCase(str2);
     }

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java

@@ -75,6 +75,13 @@ public class OSUtilsTest {
         }
     }
 
+    @Test
+    public void testGetSudoCmd() {
+        String cmd = "kill -9 1234";
+        String sudoCmd = OSUtils.getSudoCmd("test123", cmd);
+        Assert.assertEquals("sudo -u test123 " + cmd, sudoCmd);
+    }
+
     @Test
     public void exeCmd() {
         if(OSUtils.isMacOS() || !OSUtils.isWindows()){

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java

@@ -77,4 +77,22 @@ public class StringUtilsTest {
         String result4 = StringUtils.replaceNRTtoUnderline(null);
         Assert.assertNull(result4);
     }
+
+    @Test
+    public void testTrim() {
+        String trim = StringUtils.trim(null);
+        Assert.assertNull(trim);
+
+        trim = StringUtils.trim(" test ");
+        Assert.assertEquals("test", trim);
+    }
+
+    @Test
+    public void testDefaultIfBlank() {
+        String defaultStr = StringUtils.defaultIfBlank("", "defaultStr");
+        Assert.assertEquals("defaultStr", defaultStr);
+
+        defaultStr = StringUtils.defaultIfBlank("test", "defaultStr");
+        Assert.assertEquals("test", defaultStr);
+    }
 }

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java

@@ -64,6 +64,21 @@ public abstract class BaseDataSource {
      */
     private String principal;
 
+    /**
+     * java.security.krb5.conf
+     */
+    private String javaSecurityKrb5Conf;
+
+    /**
+     * login.user.keytab.username
+     */
+    private String loginUserKeytabUsername;
+
+    /**
+     * login.user.keytab.path
+     */
+    private String loginUserKeytabPath;
+
     public String getPrincipal() {
         return principal;
     }
@@ -211,4 +226,27 @@ public abstract class BaseDataSource {
         this.other = other;
     }
 
+    public String getJavaSecurityKrb5Conf() {
+        return javaSecurityKrb5Conf;
+    }
+
+    public void setJavaSecurityKrb5Conf(String javaSecurityKrb5Conf) {
+        this.javaSecurityKrb5Conf = javaSecurityKrb5Conf;
+    }
+
+    public String getLoginUserKeytabUsername() {
+        return loginUserKeytabUsername;
+    }
+
+    public void setLoginUserKeytabUsername(String loginUserKeytabUsername) {
+        this.loginUserKeytabUsername = loginUserKeytabUsername;
+    }
+
+    public String getLoginUserKeytabPath() {
+        return loginUserKeytabPath;
+    }
+
+    public void setLoginUserKeytabPath(String loginUserKeytabPath) {
+        this.loginUserKeytabPath = loginUserKeytabPath;
+    }
 }

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java

@@ -96,7 +96,7 @@ public class HiveDataSource extends BaseDataSource {
      */
     @Override
     public Connection getConnection() throws Exception {
-        CommonUtils.loadKerberosConf();
+        CommonUtils.loadKerberosConf(getJavaSecurityKrb5Conf(), getLoginUserKeytabUsername(), getLoginUserKeytabPath());
         return super.getConnection();
     }
 

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java

@@ -52,7 +52,7 @@ public class SparkDataSource extends BaseDataSource {
      */
     @Override
     public Connection getConnection() throws Exception {
-        CommonUtils.loadKerberosConf();
+        CommonUtils.loadKerberosConf(getJavaSecurityKrb5Conf(), getLoginUserKeytabUsername(), getLoginUserKeytabPath());
         return super.getConnection();
     }
 }

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java

@@ -14,11 +14,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.dao.datasource;
 
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.DbType;
 import org.apache.dolphinscheduler.common.utils.PropertyUtils;
+
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -112,7 +114,6 @@ public class BaseDataSourceTest {
         db2DataSource.setOther("charset=UTF-8");
         Assert.assertEquals("jdbc:db2://127.0.0.1:50000/test:charset=UTF-8", db2DataSource.getJdbcUrl());
 
-
     }
 
     @Test
@@ -155,10 +156,6 @@ public class BaseDataSourceTest {
         Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
         Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
 
-
     }
 
-
-
-
 }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java

@@ -344,10 +344,7 @@ public class ProcessUtils {
             }
 
             String runCmd = String.format("%s %s", Constants.SH, commandFile);
-            if (StringUtils.isNotEmpty(tenantCode)) {
-                runCmd = "sudo -u " + tenantCode + " " + runCmd;
-            }
-
+            runCmd = OSUtils.getSudoCmd(tenantCode, runCmd);
             logger.info("kill cmd:{}", runCmd);
             OSUtils.exeCmd(runCmd);
         } catch (Exception e) {
@@ -369,8 +366,8 @@ public class ProcessUtils {
                 return;
             }
 
-            String cmd = String.format("sudo kill -9 %s", getPidsStr(processId));
-
+            String cmd = String.format("kill -9 %s", getPidsStr(processId));
+            cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd);
             logger.info("process id:{}, cmd:{}", processId, cmd);
 
             OSUtils.exeCmd(cmd);

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java

@@ -115,8 +115,8 @@ public class TaskKillProcessor implements NettyRequestProcessor {
                 return Pair.of(true, appIds);
             }
 
-            String cmd = String.format("sudo kill -9 %s", ProcessUtils.getPidsStr(taskExecutionContext.getProcessId()));
-
+            String cmd = String.format("kill -9 %s", ProcessUtils.getPidsStr(taskExecutionContext.getProcessId()));
+            cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd);
             logger.info("process id:{}, cmd:{}", taskExecutionContext.getProcessId(), cmd);
 
             OSUtils.exeCmd(cmd);

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java

@@ -27,6 +27,7 @@ import org.apache.dolphinscheduler.common.thread.Stopper;
 import org.apache.dolphinscheduler.common.thread.ThreadUtils;
 import org.apache.dolphinscheduler.common.utils.HadoopUtils;
 import org.apache.dolphinscheduler.common.utils.LoggerUtils;
+import org.apache.dolphinscheduler.common.utils.OSUtils;
 import org.apache.dolphinscheduler.common.utils.StringUtils;
 import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
 import org.apache.dolphinscheduler.server.utils.ProcessUtils;
@@ -265,8 +266,8 @@ public abstract class AbstractCommandExecutor {
         if (processId != 0 && process.isAlive()) {
             try {
                 // sudo -u user command to run command
-                String cmd = String.format("sudo kill %d", processId);
-
+                String cmd = String.format("kill %d", processId);
+                cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd);
                 logger.info("soft kill task:{}, process id:{}, cmd:{}", taskExecutionContext.getTaskAppId(), processId, cmd);
 
                 Runtime.getRuntime().exec(cmd);
@@ -286,8 +287,8 @@ public abstract class AbstractCommandExecutor {
     private void hardKill(int processId) {
         if (processId != 0 && process.isAlive()) {
             try {
-                String cmd = String.format("sudo kill -9 %d", processId);
-
+                String cmd = String.format("kill -9 %d", processId);
+                cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd);
                 logger.info("hard kill task:{}, process id:{}, cmd:{}", taskExecutionContext.getTaskAppId(), processId, cmd);
 
                 Runtime.getRuntime().exec(cmd);

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java

@@ -17,12 +17,6 @@
 
 package org.apache.dolphinscheduler.server.worker.task.sql;
 
-import static org.apache.dolphinscheduler.common.Constants.HIVE_CONF;
-import static org.apache.dolphinscheduler.common.Constants.PASSWORD;
-import static org.apache.dolphinscheduler.common.Constants.SEMICOLON;
-import static org.apache.dolphinscheduler.common.Constants.USER;
-import static org.apache.dolphinscheduler.common.enums.DbType.HIVE;
-
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.CommandType;
 import org.apache.dolphinscheduler.common.enums.DbType;
@@ -33,7 +27,6 @@ import org.apache.dolphinscheduler.common.task.sql.SqlBinds;
 import org.apache.dolphinscheduler.common.task.sql.SqlParameters;
 import org.apache.dolphinscheduler.common.task.sql.SqlType;
 import org.apache.dolphinscheduler.common.utils.CollectionUtils;
-import org.apache.dolphinscheduler.common.utils.CommonUtils;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.common.utils.ParameterUtils;
 import org.apache.dolphinscheduler.common.utils.StringUtils;
@@ -50,7 +43,6 @@ import org.apache.dolphinscheduler.service.alert.AlertClientService;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
 
 import java.sql.Connection;
-import java.sql.DriverManager;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.ResultSetMetaData;
@@ -61,7 +53,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Properties;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -135,8 +126,6 @@ public class SqlTask extends AbstractTask {
                 sqlParameters.getConnParams());
         try {
             SQLTaskExecutionContext sqlTaskExecutionContext = taskExecutionContext.getSqlTaskExecutionContext();
-            // load class
-            DataSourceFactory.loadClass(DbType.valueOf(sqlParameters.getType()));
 
             // get datasource
             baseDataSource = DataSourceFactory.getDatasource(DbType.valueOf(sqlParameters.getType()),
@@ -253,10 +242,8 @@ public class SqlTask extends AbstractTask {
         PreparedStatement stmt = null;
         ResultSet resultSet = null;
         try {
-            // if upload resource is HDFS and kerberos startup
-            CommonUtils.loadKerberosConf();
             // create connection
-            connection = createConnection();
+            connection = baseDataSource.getConnection();
             // create temp function
             if (CollectionUtils.isNotEmpty(createFuncs)) {
                 createTempFunction(connection, createFuncs);
@@ -364,34 +351,6 @@ public class SqlTask extends AbstractTask {
         }
     }
 
-    /**
-     * create connection
-     *
-     * @return connection
-     * @throws Exception Exception
-     */
-    private Connection createConnection() throws Exception {
-        // if hive , load connection params if exists
-        Connection connection = null;
-        if (HIVE == DbType.valueOf(sqlParameters.getType())) {
-            Properties paramProp = new Properties();
-            paramProp.setProperty(USER, baseDataSource.getUser());
-            paramProp.setProperty(PASSWORD, baseDataSource.getPassword());
-            Map<String, String> connParamMap = CollectionUtils.stringToMap(sqlParameters.getConnParams(),
-                    SEMICOLON,
-                    HIVE_CONF);
-            paramProp.putAll(connParamMap);
-
-            connection = DriverManager.getConnection(baseDataSource.getJdbcUrl(),
-                    paramProp);
-        } else {
-            connection = DriverManager.getConnection(baseDataSource.getJdbcUrl(),
-                    baseDataSource.getUser(),
-                    baseDataSource.getPassword());
-        }
-        return connection;
-    }
-
     /**
      * close jdbc resource
      *

dolphinscheduler-server/src/main/resources/config/install_config.conf

@@ -48,35 +48,10 @@ installPath="/data1_1T/dolphinscheduler"
 # Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself
 deployUser="dolphinscheduler"
 
-
 # alert config
-# mail server host
-mailServerHost="smtp.exmail.qq.com"
-
-# mail server port
-# note: Different protocols and encryption methods correspond to different ports, when SSL/TLS is enabled, make sure the port is correct.
-mailServerPort="25"
-
-# sender
-mailSender="xxxxxxxxxx"
-
-# user
-mailUser="xxxxxxxxxx"
-
-# sender password
-# note: The mail.passwd is email service authorization code, not the email login password.
-mailPassword="xxxxxxxxxx"
-
-# TLS mail protocol support
-starttlsEnable="true"
-
-# SSL mail protocol support
-# only one of TLS and SSL can be in the true state.
-sslEnable="false"
-
-#note: sslTrust is the same as mailServerHost
-sslTrust="smtp.exmail.qq.com"
-
+# alert plugin dir
+# Note: find and load the Alert Plugin Jar from this dir.
+alertPluginDir="/data1_1T/dolphinscheduler/lib/plugin/alert"
 
 # resource storage type：HDFS,S3,NONE
 resourceStorageType="NONE"

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java

@@ -99,12 +99,13 @@ public class ProcessUtilsTest {
         try {
             when(OSUtils.exeCmd(String.format("%s -sp %d", Constants.PSTREE, 1))).thenReturn("1111");
             when(OSUtils.exeCmd(String.format("%s -p %d", Constants.PSTREE, 1))).thenReturn("1111");
-            when(OSUtils.exeCmd("sudo kill -9")).thenReturn("1111");
+            when(OSUtils.exeCmd("sudo -u tenantCode kill -9")).thenReturn("1111");
         } catch (Exception e) {
             e.printStackTrace();
         }
         taskExecutionContext.setHost("127.0.0.1:8888");
         taskExecutionContext.setLogPath("/log/1.log");
+        taskExecutionContext.setTenantCode("tenantCode");
         ProcessUtils.kill(taskExecutionContext);
         Assert.assertEquals(1, taskExecutionContext.getProcessId());
     }

dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue

@@ -84,6 +84,39 @@
             </el-input>
           </template>
         </m-list-box-f>
+        <m-list-box-f :class="{hidden:showPrincipal}">
+          <template slot="name">krb5.conf</template>
+          <template slot="content">
+            <el-input
+              type="input"
+              v-model="javaSecurityKrb5Conf"
+              size="small"
+              :placeholder="$t('Please enter the kerberos authentication parameter java.security.krb5.conf')">
+            </el-input>
+          </template>
+        </m-list-box-f>
+        <m-list-box-f :class="{hidden:showPrincipal}">
+          <template slot="name">keytab.username</template>
+          <template slot="content">
+            <el-input
+              type="input"
+              v-model="loginUserKeytabUsername"
+              size="small"
+              :placeholder="$t('Please enter the kerberos authentication parameter login.user.keytab.username')">
+            </el-input>
+          </template>
+        </m-list-box-f>
+        <m-list-box-f :class="{hidden:showPrincipal}">
+          <template slot="name">keytab.path</template>
+          <template slot="content">
+            <el-input
+              type="input"
+              v-model="loginUserKeytabPath"
+              size="small"
+              :placeholder="$t('Please enter the kerberos authentication parameter login.user.keytab.path')">
+            </el-input>
+          </template>
+        </m-list-box-f>
         <m-list-box-f>
           <template slot="name"><strong>*</strong>{{$t('User Name')}}</template>
           <template slot="content">
@@ -108,7 +141,7 @@
           </template>
         </m-list-box-f>
         <m-list-box-f>
-          <template slot="name"><strong :class="{hidden:showdDatabase}">*</strong>{{$t('Database Name')}}</template>
+          <template slot="name"><strong :class="{hidden:showDatabase}">*</strong>{{$t('Database Name')}}</template>
           <template slot="content">
             <el-input
                     type="input"
@@ -176,6 +209,12 @@
         database: '',
         // principal
         principal: '',
+        // java.security.krb5.conf
+        javaSecurityKrb5Conf: '',
+        // login.user.keytab.username
+        loginUserKeytabUsername: '',
+        // login.user.keytab.path
+        loginUserKeytabPath: '',
         // database username
         userName: '',
         // Database password
@@ -187,7 +226,7 @@
         // btn test loading
         testLoading: false,
         showPrincipal: true,
-        showdDatabase: false,
+        showDatabase: false,
         showConnectType: false,
         isShowPrincipal: true,
         prePortMapper: {},
@@ -267,6 +306,9 @@
           port: this.port,
           database: this.database,
           principal: this.principal,
+          javaSecurityKrb5Conf: this.javaSecurityKrb5Conf,
+          loginUserKeytabUsername: this.loginUserKeytabUsername,
+          loginUserKeytabPath: this.loginUserKeytabPath,
           userName: this.userName,
           password: this.password,
           connectType: this.connectType,
@@ -328,7 +370,7 @@
           return false
         }
 
-        if (!this.database && this.showdDatabase === false) {
+        if (!this.database && this.showDatabase === false) {
           this.$message.warning(`${i18n.$t('Please enter database name')}`)
           return false
         }
@@ -338,7 +380,6 @@
             return false
           }
         }
-
         return true
       },
       /**
@@ -376,6 +417,9 @@
           }, 0)
 
           this.principal = res.principal
+          this.javaSecurityKrb5Conf = res.javaSecurityKrb5Conf
+          this.loginUserKeytabUsername = res.loginUserKeytabUsername
+          this.loginUserKeytabPath = res.loginUserKeytabPath
           this.database = res.database
           this.userName = res.userName
           this.password = res.password
@@ -450,9 +494,9 @@
     watch: {
       type (value) {
         if (value === 'POSTGRESQL') {
-          this.showdDatabase = true
+          this.showDatabase = true
         } else {
-          this.showdDatabase = false
+          this.showDatabase = false
         }
 
         if (value === 'ORACLE' && !this.item.id) {

dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js

@@ -542,6 +542,9 @@ export default {
   statistics: 'Statistics',
   'select tenant': 'select tenant',
   'Please enter Principal': 'Please enter Principal',
+  'Please enter the kerberos authentication parameter java.security.krb5.conf': 'Please enter the kerberos authentication parameter java.security.krb5.conf',
+  'Please enter the kerberos authentication parameter login.user.keytab.username': 'Please enter the kerberos authentication parameter login.user.keytab.username',
+  'Please enter the kerberos authentication parameter login.user.keytab.path': 'Please enter the kerberos authentication parameter login.user.keytab.path',
   'The start time must not be the same as the end': 'The start time must not be the same as the end',
   'Startup parameter': 'Startup parameter',
   'Startup type': 'Startup type',

dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js

@@ -542,6 +542,9 @@ export default {
   statistics: '统计',
   'select tenant': '选择租户',
   'Please enter Principal': '请输入Principal',
+  'Please enter the kerberos authentication parameter java.security.krb5.conf': '请输入kerberos认证参数 java.security.krb5.conf',
+  'Please enter the kerberos authentication parameter login.user.keytab.username': '请输入kerberos认证参数 login.user.keytab.username',
+  'Please enter the kerberos authentication parameter login.user.keytab.path': '请输入kerberos认证参数 login.user.keytab.path',
   'The start time must not be the same as the end': '开始时间和结束时间不能相同',
   'Startup parameter': '启动参数',
   'Startup type': '启动类型',

install.sh

@@ -55,14 +55,8 @@ sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g
 sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/zookeeper.properties
 sed -i ${txt} "s#\#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/zookeeper.properties
 sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application-api.properties
-sed -i ${txt} "s#mail.server.host.*#mail.server.host=${mailServerHost}#g" conf/alert.properties
-sed -i ${txt} "s#mail.server.port.*#mail.server.port=${mailServerPort}#g" conf/alert.properties
-sed -i ${txt} "s#mail.sender.*#mail.sender=${mailSender}#g" conf/alert.properties
-sed -i ${txt} "s#mail.user.*#mail.user=${mailUser}#g" conf/alert.properties
-sed -i ${txt} "s#mail.passwd.*#mail.passwd=${mailPassword}#g" conf/alert.properties
-sed -i ${txt} "s#mail.smtp.starttls.enable.*#mail.smtp.starttls.enable=${starttlsEnable}#g" conf/alert.properties
-sed -i ${txt} "s#mail.smtp.ssl.trust.*#mail.smtp.ssl.trust=${sslTrust}#g" conf/alert.properties
-sed -i ${txt} "s#mail.smtp.ssl.enable.*#mail.smtp.ssl.enable=${sslEnable}#g" conf/alert.properties
+
+sed -i ${txt} "s#alert.plugin.dir.*#alert.plugin.dir=${alertPluginDir}#g" conf/alert.properties
 
 sed -i ${txt} "s#alert.listen.host.*#alert.listen.host=${alertServer}#g" conf/worker.properties