diff --git a/dolphinscheduler-alert/src/main/resources/alert.properties b/dolphinscheduler-alert/src/main/resources/alert.properties index 1e26d27fec..80ea87c2cd 100644 --- a/dolphinscheduler-alert/src/main/resources/alert.properties +++ b/dolphinscheduler-alert/src/main/resources/alert.properties @@ -21,7 +21,7 @@ #eg : Alert Server Listener port #alert.plugin.dir config the Alert Plugin dir . AlertServer while find and load the Alert Plugin Jar from this dir when deploy and start AlertServer on the server . -#eg :alert.plugin.dir=/opt/soft/spi/lib/plugin/alert +alert.plugin.dir=./lib/plugin/alert #maven.local.repository=/Users/gaojun/Documents/jianguoyun/localRepository diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java index a67ade2ba7..9fbe8f4e0a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java @@ -96,10 +96,14 @@ public class DataSourceController extends BaseController { @ApiImplicitParam(name = "host", value = "DATA_SOURCE_HOST", required = true, dataType = "String"), @ApiImplicitParam(name = "port", value = "DATA_SOURCE_PORT", required = true, dataType = "String"), @ApiImplicitParam(name = "database", value = "DATABASE_NAME", required = true, dataType = "String"), + @ApiImplicitParam(name = "principal", value = "DATA_SOURCE_PRINCIPAL", dataType = "String"), @ApiImplicitParam(name = "userName", value = "USER_NAME", required = true, dataType = "String"), @ApiImplicitParam(name = "password", value = "PASSWORD", dataType = "String"), @ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"), - @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String") + @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String"), + @ApiImplicitParam(name = "javaSecurityKrb5Conf", value = "DATA_SOURCE_KERBEROS_KRB5_CONF", dataType = "String"), + @ApiImplicitParam(name = "loginUserKeytabUsername", value = "DATA_SOURCE_KERBEROS_KEYTAB_USERNAME", dataType = "String"), + @ApiImplicitParam(name = "loginUserKeytabPath", value = "DATA_SOURCE_KERBEROS_KEYTAB_PATH", dataType = "String") }) @PostMapping(value = "/create") @ResponseStatus(HttpStatus.CREATED) @@ -115,10 +119,14 @@ public class DataSourceController extends BaseController { @RequestParam(value = "userName") String userName, @RequestParam(value = "password") String password, @RequestParam(value = "connectType") DbConnectType connectType, - @RequestParam(value = "other") String other) { + @RequestParam(value = "other") String other, + @RequestParam(value = "javaSecurityKrb5Conf", required = false) String javaSecurityKrb5Conf, + @RequestParam(value = "loginUserKeytabUsername", required = false) String loginUserKeytabUsername, + @RequestParam(value = "loginUserKeytabPath", required = false) String loginUserKeytabPath) { logger.info("login user {} create datasource name: {}, note: {}, type: {}, host: {}, port: {}, database : {}, principal: {}, userName : {}, connectType: {}, other: {}", loginUser.getUserName(), name, note, type, host, port, database, principal, userName, connectType, other); - String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other); + String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other, + javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath); return dataSourceService.createDataSource(loginUser, name, note, type, parameter); } @@ -149,10 +157,14 @@ public class DataSourceController extends BaseController { @ApiImplicitParam(name = "host", value = "DATA_SOURCE_HOST", required = true, dataType = "String"), @ApiImplicitParam(name = "port", value = "DATA_SOURCE_PORT", required = true, dataType = "String"), @ApiImplicitParam(name = "database", value = "DATABASE_NAME", required = true, dataType = "String"), + @ApiImplicitParam(name = "principal", value = "DATA_SOURCE_PRINCIPAL", dataType = "String"), @ApiImplicitParam(name = "userName", value = "USER_NAME", required = true, dataType = "String"), @ApiImplicitParam(name = "password", value = "PASSWORD", dataType = "String"), @ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"), - @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String") + @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String"), + @ApiImplicitParam(name = "javaSecurityKrb5Conf", value = "DATA_SOURCE_KERBEROS_KRB5_CONF", dataType = "String"), + @ApiImplicitParam(name = "loginUserKeytabUsername", value = "DATA_SOURCE_KERBEROS_KEYTAB_USERNAME", dataType = "String"), + @ApiImplicitParam(name = "loginUserKeytabPath", value = "DATA_SOURCE_KERBEROS_KEYTAB_PATH", dataType = "String") }) @PostMapping(value = "/update") @ResponseStatus(HttpStatus.OK) @@ -169,10 +181,14 @@ public class DataSourceController extends BaseController { @RequestParam(value = "userName") String userName, @RequestParam(value = "password") String password, @RequestParam(value = "connectType") DbConnectType connectType, - @RequestParam(value = "other") String other) { + @RequestParam(value = "other") String other, + @RequestParam(value = "javaSecurityKrb5Conf", required = false) String javaSecurityKrb5Conf, + @RequestParam(value = "loginUserKeytabUsername", required = false) String loginUserKeytabUsername, + @RequestParam(value = "loginUserKeytabPath", required = false) String loginUserKeytabPath) { logger.info("login user {} updateProcessInstance datasource name: {}, note: {}, type: {}, connectType: {}, other: {}", loginUser.getUserName(), name, note, type, connectType, other); - String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other); + String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other, + javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath); return dataSourceService.updateDataSource(id, loginUser, name, note, type, parameter); } @@ -274,10 +290,14 @@ public class DataSourceController extends BaseController { @ApiImplicitParam(name = "host", value = "DATA_SOURCE_HOST", required = true, dataType = "String"), @ApiImplicitParam(name = "port", value = "DATA_SOURCE_PORT", required = true, dataType = "String"), @ApiImplicitParam(name = "database", value = "DATABASE_NAME", required = true, dataType = "String"), + @ApiImplicitParam(name = "principal", value = "DATA_SOURCE_PRINCIPAL", dataType = "String"), @ApiImplicitParam(name = "userName", value = "USER_NAME", required = true, dataType = "String"), @ApiImplicitParam(name = "password", value = "PASSWORD", dataType = "String"), @ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"), - @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String") + @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String"), + @ApiImplicitParam(name = "javaSecurityKrb5Conf", value = "DATA_SOURCE_KERBEROS_KRB5_CONF", dataType = "String"), + @ApiImplicitParam(name = "loginUserKeytabUsername", value = "DATA_SOURCE_KERBEROS_KEYTAB_USERNAME", dataType = "String"), + @ApiImplicitParam(name = "loginUserKeytabPath", value = "DATA_SOURCE_KERBEROS_KEYTAB_PATH", dataType = "String") }) @PostMapping(value = "/connect") @ResponseStatus(HttpStatus.OK) @@ -293,10 +313,14 @@ public class DataSourceController extends BaseController { @RequestParam(value = "userName") String userName, @RequestParam(value = "password") String password, @RequestParam(value = "connectType") DbConnectType connectType, - @RequestParam(value = "other") String other) { + @RequestParam(value = "other") String other, + @RequestParam(value = "javaSecurityKrb5Conf", required = false) String javaSecurityKrb5Conf, + @RequestParam(value = "loginUserKeytabUsername", required = false) String loginUserKeytabUsername, + @RequestParam(value = "loginUserKeytabPath", required = false) String loginUserKeytabPath) { logger.info("login user {}, connect datasource: {}, note: {}, type: {}, connectType: {}, other: {}", loginUser.getUserName(), name, note, type, connectType, other); - String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other); + String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other, + javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath); return dataSourceService.checkConnection(type, parameter); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index 58bb657c6f..2ca9cbea6a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -267,6 +267,9 @@ public class DataSourceService extends BaseService { map.put(HOST, host); map.put(PORT, port); map.put(PRINCIPAL, datasourceForm.getPrincipal()); + map.put(Constants.KERBEROS_KRB5_CONF_PATH, datasourceForm.getJavaSecurityKrb5Conf()); + map.put(Constants.KERBEROS_KEY_TAB_USERNAME, datasourceForm.getLoginUserKeytabUsername()); + map.put(Constants.KERBEROS_KEY_TAB_PATH, datasourceForm.getLoginUserKeytabPath()); map.put(DATABASE, database); map.put(USER_NAME, datasourceForm.getUser()); map.put(OTHER, otherMap); @@ -424,7 +427,8 @@ public class DataSourceService extends BaseService { */ public String buildParameter(DbType type, String host, String port, String database, String principal, String userName, - String password, DbConnectType connectType, String other) { + String password, DbConnectType connectType, String other, + String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath) { String address = buildAddress(type, host, port, connectType); Map parameterMap = new LinkedHashMap(6); @@ -467,6 +471,9 @@ public class DataSourceService extends BaseService { if (CommonUtils.getKerberosStartupState() && (type == DbType.HIVE || type == DbType.SPARK)) { parameterMap.put(Constants.PRINCIPAL, principal); + parameterMap.put(Constants.KERBEROS_KRB5_CONF_PATH, javaSecurityKrb5Conf); + parameterMap.put(Constants.KERBEROS_KEY_TAB_USERNAME, loginUserKeytabUsername); + parameterMap.put(Constants.KERBEROS_KEY_TAB_PATH, loginUserKeytabPath); } Map map = JSONUtils.toMap(other); diff --git a/dolphinscheduler-api/src/main/resources/i18n/messages.properties b/dolphinscheduler-api/src/main/resources/i18n/messages.properties index cae8e5fa59..62d3615664 100644 --- a/dolphinscheduler-api/src/main/resources/i18n/messages.properties +++ b/dolphinscheduler-api/src/main/resources/i18n/messages.properties @@ -125,6 +125,10 @@ TENANT_CODE=os tenant code QUEUE_NAME=queue name PASSWORD=password DATA_SOURCE_OTHER=jdbc connection params, format:{"key1":"value1",...} +DATA_SOURCE_PRINCIPAL=principal +DATA_SOURCE_KERBEROS_KRB5_CONF=the kerberos authentication parameter java.security.krb5.conf +DATA_SOURCE_KERBEROS_KEYTAB_USERNAME=the kerberos authentication parameter login.user.keytab.username +DATA_SOURCE_KERBEROS_KEYTAB_PATH=the kerberos authentication parameter login.user.keytab.path PROJECT_TAG=project related operation CREATE_PROJECT_NOTES=create project PROJECT_DESC=project description diff --git a/dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties b/dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties index 18b7a10885..74fc4d01b0 100644 --- a/dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties +++ b/dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties @@ -125,6 +125,10 @@ TENANT_CODE=os tenant code QUEUE_NAME=queue name PASSWORD=password DATA_SOURCE_OTHER=jdbc connection params, format:{"key1":"value1",...} +DATA_SOURCE_PRINCIPAL=principal +DATA_SOURCE_KERBEROS_KRB5_CONF=the kerberos authentication parameter java.security.krb5.conf +DATA_SOURCE_KERBEROS_KEYTAB_USERNAME=the kerberos authentication parameter login.user.keytab.username +DATA_SOURCE_KERBEROS_KEYTAB_PATH=the kerberos authentication parameter login.user.keytab.path PROJECT_TAG=project related operation CREATE_PROJECT_NOTES=create project PROJECT_DESC=project description diff --git a/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties b/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties index a333ef1fa5..f1e8ebefc3 100644 --- a/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties +++ b/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties @@ -119,6 +119,10 @@ TENANT_CODE=操作系统租户 QUEUE_NAME=队列名 PASSWORD=密码 DATA_SOURCE_OTHER=jdbc连接参数，格式为:{"key1":"value1",...} +DATA_SOURCE_PRINCIPAL=principal +DATA_SOURCE_KERBEROS_KRB5_CONF=kerberos认证参数 java.security.krb5.conf +DATA_SOURCE_KERBEROS_KEYTAB_USERNAME=kerberos认证参数 login.user.keytab.username +DATA_SOURCE_KERBEROS_KEYTAB_PATH=kerberos认证参数 login.user.keytab.path PROJECT_TAG=项目相关操作 CREATE_PROJECT_NOTES=创建项目 PROJECT_DESC=项目描述 diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java index 84ccd2e0a3..13eb1b9c2e 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbConnectType; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; @@ -51,7 +52,7 @@ import org.powermock.modules.junit4.PowerMockRunner; @RunWith(PowerMockRunner.class) @PowerMockIgnore({"sun.security.*", "javax.net.*"}) -@PrepareForTest({DataSourceFactory.class}) +@PrepareForTest({DataSourceFactory.class, CommonUtils.class}) public class DataSourceServiceTest { @@ -68,7 +69,7 @@ public class DataSourceServiceTest { String dataSourceName = "dataSource01"; String dataSourceDesc = "test dataSource"; DbType dataSourceType = DbType.POSTGRESQL; - String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null); + String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null, null, null, null); // data source exits List dataSourceList = new ArrayList<>(); @@ -110,7 +111,7 @@ public class DataSourceServiceTest { String dataSourceName = "dataSource01"; String dataSourceDesc = "test dataSource"; DbType dataSourceType = DbType.POSTGRESQL; - String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null); + String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null, null, null, null); // data source not exits PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(null); @@ -274,24 +275,35 @@ public class DataSourceServiceTest { @Test public void buildParameter() { String param = dataSourceService.buildParameter(DbType.ORACLE, "192.168.9.1", "1521", "im" - , "", "test", "test", DbConnectType.ORACLE_SERVICE_NAME, ""); + , "", "test", "test", DbConnectType.ORACLE_SERVICE_NAME, "", "", "",""); String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\"," + "\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}"; Assert.assertEquals(expected, param); + + PowerMockito.mockStatic(CommonUtils.class); + PowerMockito.when(CommonUtils.getKerberosStartupState()).thenReturn(true); + PowerMockito.when(CommonUtils.encodePassword(Mockito.anyString())).thenReturn("test"); + param = dataSourceService.buildParameter(DbType.HIVE, "192.168.9.1", "10000", "im" + , "hive/hdfs-mycluster@ESZ.COM", "test", "test", null, "", "/opt/krb5.conf", "test2/hdfs-mycluster@ESZ.COM", "/opt/hdfs.headless.keytab"); + expected = "{\"type\":null,\"address\":\"jdbc:hive2://192.168.9.1:10000\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:hive2://192.168.9.1:10000/im;principal=hive/hdfs-mycluster@ESZ.COM\"," + + "\"user\":\"test\",\"password\":\"test\",\"principal\":\"hive/hdfs-mycluster@ESZ.COM\",\"javaSecurityKrb5Conf\":\"/opt/krb5.conf\"," + + "\"loginUserKeytabUsername\":\"test2/hdfs-mycluster@ESZ.COM\",\"loginUserKeytabPath\":\"/opt/hdfs.headless.keytab\"}"; + Assert.assertEquals(expected, param); + } @Test public void buildParameterWithDecodePassword() { PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true"); String param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im" - , "", "test", "123456", null, ""); + , "", "test", "123456", null, "", "", "", ""); String expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\"," + "\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}"; Assert.assertEquals(expected, param); PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false"); param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im" - , "", "test", "123456", null, ""); + , "", "test", "123456", null, "", "", "", ""); expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"123456\"}"; Assert.assertEquals(expected, param); } @@ -316,7 +328,7 @@ public class DataSourceServiceTest { @Test public void testCheckConnection() throws Exception { DbType dataSourceType = DbType.POSTGRESQL; - String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null); + String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null, null, null, null); PowerMockito.mockStatic(DataSourceFactory.class); PowerMockito.when(DataSourceFactory.getDatasource(Mockito.any(), Mockito.anyString())).thenReturn(null); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 1252311b97..0520520621 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -956,7 +956,9 @@ public final class Constants { public static final String PRINCIPAL = "principal"; public static final String OTHER = "other"; public static final String ORACLE_DB_CONNECT_TYPE = "connectType"; - + public static final String KERBEROS_KRB5_CONF_PATH = "javaSecurityKrb5Conf"; + public static final String KERBEROS_KEY_TAB_USERNAME = "loginUserKeytabUsername"; + public static final String KERBEROS_KEY_TAB_PATH = "loginUserKeytabPath"; /** * session timeout diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java index 45c5aa2c93..cf307b402f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java @@ -86,13 +86,26 @@ public class CommonUtils { * @throws Exception errors */ public static void loadKerberosConf() throws Exception { + loadKerberosConf(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), + PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), + PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)); + } + + /** + * load kerberos configuration + * @param javaSecurityKrb5Conf javaSecurityKrb5Conf + * @param loginUserKeytabUsername loginUserKeytabUsername + * @param loginUserKeytabPath loginUserKeytabPath + * @throws Exception errors + */ + public static void loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath) throws Exception { if (CommonUtils.getKerberosStartupState()) { - System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); + System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, StringUtils.defaultIfBlank(javaSecurityKrb5Conf, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH))); Configuration configuration = new Configuration(); configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, Constants.KERBEROS); UserGroupInformation.setConfiguration(configuration); - UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), - PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)); + UserGroupInformation.loginUserFromKeytab(StringUtils.defaultIfBlank(loginUserKeytabUsername, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)), + StringUtils.defaultIfBlank(loginUserKeytabPath, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH))); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index b48f2d30cc..768c0f654a 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -389,6 +389,16 @@ public class OSUtils { return null; } + /** + * get sudo command + * @param tenantCode tenantCode + * @param command command + * @return result of sudo execute command + */ + public static String getSudoCmd(String tenantCode, String command) { + return StringUtils.isEmpty(tenantCode) ? command : "sudo -u " + tenantCode + " " + command; + } + /** * Execute the corresponding command of Linux or Windows * diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java index 362c6130f4..6bed928e14 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java @@ -62,6 +62,10 @@ public class StringUtils { return str == null ? null : str.trim(); } + public static String defaultIfBlank(String str, String defaultStr) { + return isBlank(str) ? defaultStr : str; + } + public static boolean equalsIgnoreCase(String str1, String str2) { return str1 == null ? str2 == null : str1.equalsIgnoreCase(str2); } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java index 8f40bccdc8..d005010f93 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java @@ -75,6 +75,13 @@ public class OSUtilsTest { } } + @Test + public void testGetSudoCmd() { + String cmd = "kill -9 1234"; + String sudoCmd = OSUtils.getSudoCmd("test123", cmd); + Assert.assertEquals("sudo -u test123 " + cmd, sudoCmd); + } + @Test public void exeCmd() { if(OSUtils.isMacOS() || !OSUtils.isWindows()){ diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java index f67e89e7e2..74b96aff02 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java @@ -77,4 +77,22 @@ public class StringUtilsTest { String result4 = StringUtils.replaceNRTtoUnderline(null); Assert.assertNull(result4); } + + @Test + public void testTrim() { + String trim = StringUtils.trim(null); + Assert.assertNull(trim); + + trim = StringUtils.trim(" test "); + Assert.assertEquals("test", trim); + } + + @Test + public void testDefaultIfBlank() { + String defaultStr = StringUtils.defaultIfBlank("", "defaultStr"); + Assert.assertEquals("defaultStr", defaultStr); + + defaultStr = StringUtils.defaultIfBlank("test", "defaultStr"); + Assert.assertEquals("test", defaultStr); + } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java index bb3825fbf9..9bbbf32a6f 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java @@ -64,6 +64,21 @@ public abstract class BaseDataSource { */ private String principal; + /** + * java.security.krb5.conf + */ + private String javaSecurityKrb5Conf; + + /** + * login.user.keytab.username + */ + private String loginUserKeytabUsername; + + /** + * login.user.keytab.path + */ + private String loginUserKeytabPath; + public String getPrincipal() { return principal; } @@ -211,4 +226,27 @@ public abstract class BaseDataSource { this.other = other; } + public String getJavaSecurityKrb5Conf() { + return javaSecurityKrb5Conf; + } + + public void setJavaSecurityKrb5Conf(String javaSecurityKrb5Conf) { + this.javaSecurityKrb5Conf = javaSecurityKrb5Conf; + } + + public String getLoginUserKeytabUsername() { + return loginUserKeytabUsername; + } + + public void setLoginUserKeytabUsername(String loginUserKeytabUsername) { + this.loginUserKeytabUsername = loginUserKeytabUsername; + } + + public String getLoginUserKeytabPath() { + return loginUserKeytabPath; + } + + public void setLoginUserKeytabPath(String loginUserKeytabPath) { + this.loginUserKeytabPath = loginUserKeytabPath; + } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java index 3c087e7f31..bcf1cdf3d2 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java @@ -96,7 +96,7 @@ public class HiveDataSource extends BaseDataSource { */ @Override public Connection getConnection() throws Exception { - CommonUtils.loadKerberosConf(); + CommonUtils.loadKerberosConf(getJavaSecurityKrb5Conf(), getLoginUserKeytabUsername(), getLoginUserKeytabPath()); return super.getConnection(); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java index 207ed43942..df17bb564c 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java @@ -52,7 +52,7 @@ public class SparkDataSource extends BaseDataSource { */ @Override public Connection getConnection() throws Exception { - CommonUtils.loadKerberosConf(); + CommonUtils.loadKerberosConf(getJavaSecurityKrb5Conf(), getLoginUserKeytabUsername(), getLoginUserKeytabPath()); return super.getConnection(); } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java index 9250e500d1..1d02fd4fe3 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java @@ -14,151 +14,148 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.utils.PropertyUtils; + import org.junit.Assert; import org.junit.Test; public class BaseDataSourceTest { - @Test - public void testDriverClassSelector() { - String mysqlDriverClass = new MySQLDataSource().driverClassSelector(); - Assert.assertEquals(Constants.COM_MYSQL_JDBC_DRIVER, mysqlDriverClass); - - String clickHouseDriverClass = new ClickHouseDataSource().driverClassSelector(); - Assert.assertEquals(Constants.COM_CLICKHOUSE_JDBC_DRIVER, clickHouseDriverClass); - - String db2ServerDriverClass = new DB2ServerDataSource().driverClassSelector(); - Assert.assertEquals(Constants.COM_DB2_JDBC_DRIVER, db2ServerDriverClass); - - String oracleDriverClass = new OracleDataSource().driverClassSelector(); - Assert.assertEquals(Constants.COM_ORACLE_JDBC_DRIVER, oracleDriverClass); - - String postgreDriverClass = new PostgreDataSource().driverClassSelector(); - Assert.assertEquals(Constants.ORG_POSTGRESQL_DRIVER, postgreDriverClass); - - String sqlServerDriverClass = new SQLServerDataSource().driverClassSelector(); - Assert.assertEquals(Constants.COM_SQLSERVER_JDBC_DRIVER, sqlServerDriverClass); - - String hiveDriverClass = new HiveDataSource().driverClassSelector(); - Assert.assertEquals(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER, hiveDriverClass); - - String sparkDriverClass = new SparkDataSource().driverClassSelector(); - Assert.assertEquals(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER, sparkDriverClass); - } - - @Test - public void testGetJdbcUrl() { - BaseDataSource hiveDataSource = new HiveDataSource(); - hiveDataSource.setAddress("jdbc:hive2://127.0.0.1:10000"); - hiveDataSource.setDatabase("test"); - hiveDataSource.setPassword("123456"); - hiveDataSource.setUser("test"); - Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test", hiveDataSource.getJdbcUrl()); - //set principal - hiveDataSource.setPrincipal("hive/test.com@TEST.COM"); - Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test;principal=hive/test.com@TEST.COM", - hiveDataSource.getJdbcUrl()); - //set fake other - hiveDataSource.setOther("charset=UTF-8"); - Assert.assertEquals( - "jdbc:hive2://127.0.0.1:10000/test;principal=hive/test.com@TEST.COM;charset=UTF-8", - hiveDataSource.getJdbcUrl()); - - BaseDataSource clickHouseDataSource = new ClickHouseDataSource(); - clickHouseDataSource.setAddress("jdbc:clickhouse://127.0.0.1:8123"); - clickHouseDataSource.setDatabase("test"); - clickHouseDataSource.setPassword("123456"); - clickHouseDataSource.setUser("test"); - Assert.assertEquals("jdbc:clickhouse://127.0.0.1:8123/test", clickHouseDataSource.getJdbcUrl()); - //set fake principal - clickHouseDataSource.setPrincipal("fake principal"); - Assert.assertEquals("jdbc:clickhouse://127.0.0.1:8123/test", clickHouseDataSource.getJdbcUrl()); - //set fake other - clickHouseDataSource.setOther("charset=UTF-8"); - Assert.assertEquals("jdbc:clickhouse://127.0.0.1:8123/test?charset=UTF-8", - clickHouseDataSource.getJdbcUrl()); - - BaseDataSource sqlServerDataSource = new SQLServerDataSource(); - sqlServerDataSource.setAddress("jdbc:sqlserver://127.0.0.1:1433"); - sqlServerDataSource.setDatabase("test"); - sqlServerDataSource.setPassword("123456"); - sqlServerDataSource.setUser("test"); - Assert.assertEquals("jdbc:sqlserver://127.0.0.1:1433;databaseName=test", - sqlServerDataSource.getJdbcUrl()); - //set fake principal - sqlServerDataSource.setPrincipal("fake principal"); - Assert.assertEquals("jdbc:sqlserver://127.0.0.1:1433;databaseName=test", - sqlServerDataSource.getJdbcUrl()); - //set fake other - sqlServerDataSource.setOther("charset=UTF-8"); - Assert.assertEquals("jdbc:sqlserver://127.0.0.1:1433;databaseName=test;charset=UTF-8", - sqlServerDataSource.getJdbcUrl()); - - BaseDataSource db2DataSource = new DB2ServerDataSource(); - db2DataSource.setAddress("jdbc:db2://127.0.0.1:50000"); - db2DataSource.setDatabase("test"); - db2DataSource.setPassword("123456"); - db2DataSource.setUser("test"); - Assert.assertEquals("jdbc:db2://127.0.0.1:50000/test", db2DataSource.getJdbcUrl()); - //set fake principal - db2DataSource.setPrincipal("fake principal"); - Assert.assertEquals("jdbc:db2://127.0.0.1:50000/test", db2DataSource.getJdbcUrl()); - //set fake other - db2DataSource.setOther("charset=UTF-8"); - Assert.assertEquals("jdbc:db2://127.0.0.1:50000/test:charset=UTF-8", db2DataSource.getJdbcUrl()); - - - } - - @Test - public void testGetPassword() { - BaseDataSource dataSource = new BaseDataSource() { - @Override - public String driverClassSelector() { - return null; - } - - @Override - public DbType dbTypeSelector() { - return null; - } - }; - - String password= ""; - dataSource.setPassword(password); - Assert.assertEquals("", dataSource.getPassword()); - password= "IUAjJCVeJipNVEl6TkRVMg=="; - dataSource.setPassword(password); - Assert.assertNotNull(dataSource.getPassword()); - Assert.assertNotNull(dataSource.getPassword()); - - dataSource.setPassword(password); - PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true"); - Assert.assertEquals("123456", dataSource.getPassword()); - - dataSource.setPassword(password); - Assert.assertEquals("123456", dataSource.getPassword()); - Assert.assertEquals("123456", dataSource.getPassword()); - Assert.assertEquals("123456", dataSource.getPassword()); - - dataSource.setPassword(password); - PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false"); - Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); - - dataSource.setPassword(password); - Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); - Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); - Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); - - - } - - - + @Test + public void testDriverClassSelector() { + String mysqlDriverClass = new MySQLDataSource().driverClassSelector(); + Assert.assertEquals(Constants.COM_MYSQL_JDBC_DRIVER, mysqlDriverClass); + + String clickHouseDriverClass = new ClickHouseDataSource().driverClassSelector(); + Assert.assertEquals(Constants.COM_CLICKHOUSE_JDBC_DRIVER, clickHouseDriverClass); + + String db2ServerDriverClass = new DB2ServerDataSource().driverClassSelector(); + Assert.assertEquals(Constants.COM_DB2_JDBC_DRIVER, db2ServerDriverClass); + + String oracleDriverClass = new OracleDataSource().driverClassSelector(); + Assert.assertEquals(Constants.COM_ORACLE_JDBC_DRIVER, oracleDriverClass); + + String postgreDriverClass = new PostgreDataSource().driverClassSelector(); + Assert.assertEquals(Constants.ORG_POSTGRESQL_DRIVER, postgreDriverClass); + + String sqlServerDriverClass = new SQLServerDataSource().driverClassSelector(); + Assert.assertEquals(Constants.COM_SQLSERVER_JDBC_DRIVER, sqlServerDriverClass); + + String hiveDriverClass = new HiveDataSource().driverClassSelector(); + Assert.assertEquals(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER, hiveDriverClass); + + String sparkDriverClass = new SparkDataSource().driverClassSelector(); + Assert.assertEquals(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER, sparkDriverClass); + } + + @Test + public void testGetJdbcUrl() { + BaseDataSource hiveDataSource = new HiveDataSource(); + hiveDataSource.setAddress("jdbc:hive2://127.0.0.1:10000"); + hiveDataSource.setDatabase("test"); + hiveDataSource.setPassword("123456"); + hiveDataSource.setUser("test"); + Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test", hiveDataSource.getJdbcUrl()); + //set principal + hiveDataSource.setPrincipal("hive/test.com@TEST.COM"); + Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test;principal=hive/test.com@TEST.COM", + hiveDataSource.getJdbcUrl()); + //set fake other + hiveDataSource.setOther("charset=UTF-8"); + Assert.assertEquals( + "jdbc:hive2://127.0.0.1:10000/test;principal=hive/test.com@TEST.COM;charset=UTF-8", + hiveDataSource.getJdbcUrl()); + + BaseDataSource clickHouseDataSource = new ClickHouseDataSource(); + clickHouseDataSource.setAddress("jdbc:clickhouse://127.0.0.1:8123"); + clickHouseDataSource.setDatabase("test"); + clickHouseDataSource.setPassword("123456"); + clickHouseDataSource.setUser("test"); + Assert.assertEquals("jdbc:clickhouse://127.0.0.1:8123/test", clickHouseDataSource.getJdbcUrl()); + //set fake principal + clickHouseDataSource.setPrincipal("fake principal"); + Assert.assertEquals("jdbc:clickhouse://127.0.0.1:8123/test", clickHouseDataSource.getJdbcUrl()); + //set fake other + clickHouseDataSource.setOther("charset=UTF-8"); + Assert.assertEquals("jdbc:clickhouse://127.0.0.1:8123/test?charset=UTF-8", + clickHouseDataSource.getJdbcUrl()); + + BaseDataSource sqlServerDataSource = new SQLServerDataSource(); + sqlServerDataSource.setAddress("jdbc:sqlserver://127.0.0.1:1433"); + sqlServerDataSource.setDatabase("test"); + sqlServerDataSource.setPassword("123456"); + sqlServerDataSource.setUser("test"); + Assert.assertEquals("jdbc:sqlserver://127.0.0.1:1433;databaseName=test", + sqlServerDataSource.getJdbcUrl()); + //set fake principal + sqlServerDataSource.setPrincipal("fake principal"); + Assert.assertEquals("jdbc:sqlserver://127.0.0.1:1433;databaseName=test", + sqlServerDataSource.getJdbcUrl()); + //set fake other + sqlServerDataSource.setOther("charset=UTF-8"); + Assert.assertEquals("jdbc:sqlserver://127.0.0.1:1433;databaseName=test;charset=UTF-8", + sqlServerDataSource.getJdbcUrl()); + + BaseDataSource db2DataSource = new DB2ServerDataSource(); + db2DataSource.setAddress("jdbc:db2://127.0.0.1:50000"); + db2DataSource.setDatabase("test"); + db2DataSource.setPassword("123456"); + db2DataSource.setUser("test"); + Assert.assertEquals("jdbc:db2://127.0.0.1:50000/test", db2DataSource.getJdbcUrl()); + //set fake principal + db2DataSource.setPrincipal("fake principal"); + Assert.assertEquals("jdbc:db2://127.0.0.1:50000/test", db2DataSource.getJdbcUrl()); + //set fake other + db2DataSource.setOther("charset=UTF-8"); + Assert.assertEquals("jdbc:db2://127.0.0.1:50000/test:charset=UTF-8", db2DataSource.getJdbcUrl()); + + } + + @Test + public void testGetPassword() { + BaseDataSource dataSource = new BaseDataSource() { + @Override + public String driverClassSelector() { + return null; + } + + @Override + public DbType dbTypeSelector() { + return null; + } + }; + + String password = ""; + dataSource.setPassword(password); + Assert.assertEquals("", dataSource.getPassword()); + password = "IUAjJCVeJipNVEl6TkRVMg=="; + dataSource.setPassword(password); + Assert.assertNotNull(dataSource.getPassword()); + Assert.assertNotNull(dataSource.getPassword()); + + dataSource.setPassword(password); + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true"); + Assert.assertEquals("123456", dataSource.getPassword()); + + dataSource.setPassword(password); + Assert.assertEquals("123456", dataSource.getPassword()); + Assert.assertEquals("123456", dataSource.getPassword()); + Assert.assertEquals("123456", dataSource.getPassword()); + + dataSource.setPassword(password); + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false"); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + + dataSource.setPassword(password); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + + } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java index 6c54c790fb..3f60140e9d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java @@ -344,10 +344,7 @@ public class ProcessUtils { } String runCmd = String.format("%s %s", Constants.SH, commandFile); - if (StringUtils.isNotEmpty(tenantCode)) { - runCmd = "sudo -u " + tenantCode + " " + runCmd; - } - + runCmd = OSUtils.getSudoCmd(tenantCode, runCmd); logger.info("kill cmd:{}", runCmd); OSUtils.exeCmd(runCmd); } catch (Exception e) { @@ -369,8 +366,8 @@ public class ProcessUtils { return; } - String cmd = String.format("sudo kill -9 %s", getPidsStr(processId)); - + String cmd = String.format("kill -9 %s", getPidsStr(processId)); + cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd); logger.info("process id:{}, cmd:{}", processId, cmd); OSUtils.exeCmd(cmd); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java index 45268e6d86..b41c189eda 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java @@ -115,8 +115,8 @@ public class TaskKillProcessor implements NettyRequestProcessor { return Pair.of(true, appIds); } - String cmd = String.format("sudo kill -9 %s", ProcessUtils.getPidsStr(taskExecutionContext.getProcessId())); - + String cmd = String.format("kill -9 %s", ProcessUtils.getPidsStr(taskExecutionContext.getProcessId())); + cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd); logger.info("process id:{}, cmd:{}", taskExecutionContext.getProcessId(), cmd); OSUtils.exeCmd(cmd); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index da5c0e6980..443bd319ed 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -27,6 +27,7 @@ import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.utils.ProcessUtils; @@ -265,8 +266,8 @@ public abstract class AbstractCommandExecutor { if (processId != 0 && process.isAlive()) { try { // sudo -u user command to run command - String cmd = String.format("sudo kill %d", processId); - + String cmd = String.format("kill %d", processId); + cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd); logger.info("soft kill task:{}, process id:{}, cmd:{}", taskExecutionContext.getTaskAppId(), processId, cmd); Runtime.getRuntime().exec(cmd); @@ -286,8 +287,8 @@ public abstract class AbstractCommandExecutor { private void hardKill(int processId) { if (processId != 0 && process.isAlive()) { try { - String cmd = String.format("sudo kill -9 %d", processId); - + String cmd = String.format("kill -9 %d", processId); + cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd); logger.info("hard kill task:{}, process id:{}, cmd:{}", taskExecutionContext.getTaskAppId(), processId, cmd); Runtime.getRuntime().exec(cmd); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index 7aabfce49d..4c328edbe0 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -17,12 +17,6 @@ package org.apache.dolphinscheduler.server.worker.task.sql; -import static org.apache.dolphinscheduler.common.Constants.HIVE_CONF; -import static org.apache.dolphinscheduler.common.Constants.PASSWORD; -import static org.apache.dolphinscheduler.common.Constants.SEMICOLON; -import static org.apache.dolphinscheduler.common.Constants.USER; -import static org.apache.dolphinscheduler.common.enums.DbType.HIVE; - import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DbType; @@ -33,7 +27,6 @@ import org.apache.dolphinscheduler.common.task.sql.SqlBinds; import org.apache.dolphinscheduler.common.task.sql.SqlParameters; import org.apache.dolphinscheduler.common.task.sql.SqlType; import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; @@ -50,7 +43,6 @@ import org.apache.dolphinscheduler.service.alert.AlertClientService; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import java.sql.Connection; -import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.ResultSetMetaData; @@ -61,7 +53,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Properties; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -135,8 +126,6 @@ public class SqlTask extends AbstractTask { sqlParameters.getConnParams()); try { SQLTaskExecutionContext sqlTaskExecutionContext = taskExecutionContext.getSqlTaskExecutionContext(); - // load class - DataSourceFactory.loadClass(DbType.valueOf(sqlParameters.getType())); // get datasource baseDataSource = DataSourceFactory.getDatasource(DbType.valueOf(sqlParameters.getType()), @@ -253,10 +242,8 @@ public class SqlTask extends AbstractTask { PreparedStatement stmt = null; ResultSet resultSet = null; try { - // if upload resource is HDFS and kerberos startup - CommonUtils.loadKerberosConf(); // create connection - connection = createConnection(); + connection = baseDataSource.getConnection(); // create temp function if (CollectionUtils.isNotEmpty(createFuncs)) { createTempFunction(connection, createFuncs); @@ -364,34 +351,6 @@ public class SqlTask extends AbstractTask { } } - /** - * create connection - * - * @return connection - * @throws Exception Exception - */ - private Connection createConnection() throws Exception { - // if hive , load connection params if exists - Connection connection = null; - if (HIVE == DbType.valueOf(sqlParameters.getType())) { - Properties paramProp = new Properties(); - paramProp.setProperty(USER, baseDataSource.getUser()); - paramProp.setProperty(PASSWORD, baseDataSource.getPassword()); - Map connParamMap = CollectionUtils.stringToMap(sqlParameters.getConnParams(), - SEMICOLON, - HIVE_CONF); - paramProp.putAll(connParamMap); - - connection = DriverManager.getConnection(baseDataSource.getJdbcUrl(), - paramProp); - } else { - connection = DriverManager.getConnection(baseDataSource.getJdbcUrl(), - baseDataSource.getUser(), - baseDataSource.getPassword()); - } - return connection; - } - /** * close jdbc resource * diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf index a20fa66087..f3ed554a88 100755 --- a/dolphinscheduler-server/src/main/resources/config/install_config.conf +++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf @@ -48,35 +48,10 @@ installPath="/data1_1T/dolphinscheduler" # Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself deployUser="dolphinscheduler" - # alert config -# mail server host -mailServerHost="smtp.exmail.qq.com" - -# mail server port -# note: Different protocols and encryption methods correspond to different ports, when SSL/TLS is enabled, make sure the port is correct. -mailServerPort="25" - -# sender -mailSender="xxxxxxxxxx" - -# user -mailUser="xxxxxxxxxx" - -# sender password -# note: The mail.passwd is email service authorization code, not the email login password. -mailPassword="xxxxxxxxxx" - -# TLS mail protocol support -starttlsEnable="true" - -# SSL mail protocol support -# only one of TLS and SSL can be in the true state. -sslEnable="false" - -#note: sslTrust is the same as mailServerHost -sslTrust="smtp.exmail.qq.com" - +# alert plugin dir +# Note: find and load the Alert Plugin Jar from this dir. +alertPluginDir="/data1_1T/dolphinscheduler/lib/plugin/alert" # resource storage type：HDFS,S3,NONE resourceStorageType="NONE" diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java index 4d1ed27619..2f25e3b893 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java @@ -99,12 +99,13 @@ public class ProcessUtilsTest { try { when(OSUtils.exeCmd(String.format("%s -sp %d", Constants.PSTREE, 1))).thenReturn("1111"); when(OSUtils.exeCmd(String.format("%s -p %d", Constants.PSTREE, 1))).thenReturn("1111"); - when(OSUtils.exeCmd("sudo kill -9")).thenReturn("1111"); + when(OSUtils.exeCmd("sudo -u tenantCode kill -9")).thenReturn("1111"); } catch (Exception e) { e.printStackTrace(); } taskExecutionContext.setHost("127.0.0.1:8888"); taskExecutionContext.setLogPath("/log/1.log"); + taskExecutionContext.setTenantCode("tenantCode"); ProcessUtils.kill(taskExecutionContext); Assert.assertEquals(1, taskExecutionContext.getProcessId()); } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue b/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue index b329831269..d188b9381c 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue @@ -84,6 +84,39 @@ + + + + + + + + + + + + - +