diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index 892d53bcbf..41374f4478 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -527,7 +527,7 @@ public class DataSourceService extends BaseService{ parameterMap.put(Constants.DATABASE, database); parameterMap.put(Constants.JDBC_URL, jdbcUrl); parameterMap.put(Constants.USER, userName); - parameterMap.put(Constants.PASSWORD, password); + parameterMap.put(Constants.PASSWORD, CommonUtils.encodePassword(password)); if (CommonUtils.getKerberosStartupState() && (type == DbType.HIVE || type == DbType.SPARK)){ parameterMap.put(Constants.PRINCIPAL,principal); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java index 00e5275e80..c185868cde 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbConnectType; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; @@ -103,7 +104,28 @@ public class DataSourceServiceTest { public void buildParameter(){ String param = dataSourceService.buildParameter("","", DbType.ORACLE, "192.168.9.1","1521","im" ,"","test","test", DbConnectType.ORACLE_SERVICE_NAME,""); - String expected = "{\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}"; + String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}"; Assert.assertEquals(expected, param); } + + @Test + public void buildParameterWithDecodePassword(){ + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true"); + String param = dataSourceService.buildParameter("name","desc", DbType.MYSQL, "192.168.9.1","1521","im" + ,"","test","123456", null,""); + String expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}"; + Assert.assertEquals(expected, param); + + + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false"); + param = dataSourceService.buildParameter("name","desc", DbType.MYSQL, "192.168.9.1","1521","im" + ,"","test","123456", null,""); + expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"123456\"}"; + Assert.assertEquals(expected, param); + } + + + + + } \ No newline at end of file diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index c37f544f9a..4cb09a1a56 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -991,4 +991,10 @@ public final class Constants { public static final String DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE = "dolphin.scheduler.network.interface.preferred"; + /** + * datasource encryption salt + */ + public static final String DATASOURCE_ENCRYPTION_SALT_DEFAULT = "!@#$%^&*"; + public static final String DATASOURCE_ENCRYPTION_ENABLE = "datasource.encryption.enable"; + public static final String DATASOURCE_ENCRYPTION_SALT = "datasource.encryption.salt"; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java index 0c23d5c2ae..6722c23037 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; +import org.apache.commons.codec.binary.Base64; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.hadoop.conf.Configuration; @@ -23,8 +24,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.net.URL; +import java.nio.charset.StandardCharsets; /** * common utils @@ -32,6 +33,8 @@ import java.net.URL; public class CommonUtils { private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class); + private static final Base64 BASE64 = new Base64(); + private CommonUtils() { throw new IllegalStateException("CommonUtils class"); } @@ -90,4 +93,45 @@ public class CommonUtils { PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)); } } + + /** + * encode password + * @param password + * @return + */ + public static String encodePassword(String password) { + if(StringUtils.isEmpty(password)){return StringUtils.EMPTY; } + //if encryption is not turned on, return directly + boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false); + if ( !encryptionEnable){ return password; } + + // Using Base64 + salt to process password + String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); + String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8))) ; + return new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8))); + } + + /** + * decode password + * @param password + * @return + */ + public static String decodePassword(String password) { + if(StringUtils.isEmpty(password)){return StringUtils.EMPTY ; } + + //if encryption is not turned on, return directly + boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false); + if ( !encryptionEnable){ return password; } + + // Using Base64 + salt to process password + String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); + String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8) ; + if(!passwordWithSalt.startsWith(salt)){ + logger.warn("There is a password and salt mismatch: {} ",password); + return password; + } + return new String(BASE64.decode(passwordWithSalt.substring(salt.length())), StandardCharsets.UTF_8) ; + } + + } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index 8f0ee327c1..895270766c 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -250,4 +250,14 @@ public class PropertyUtils { } return matchedProperties; } + + /** + * + * @param key + * @param value + */ + public static void setValue(String key, String value) { + properties.setProperty(key,value); + } + } diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 4a9baae030..a75f964fa2 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -67,4 +67,8 @@ yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s development.state=false # kerberos tgt expire time, unit is hours -kerberos.expire.time=2 \ No newline at end of file +kerberos.expire.time=2 + +# datasource encryption salt +datasource.encryption.enable=false +datasource.encryption.salt=!@#$%^&* diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java index c720013125..cb038e9503 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; +import org.apache.dolphinscheduler.common.Constants; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -89,4 +90,42 @@ public class CommonUtilsTest { } Assert.assertTrue(true); } + + @Test + public void encodePassword() { + + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true"); + + Assert.assertEquals("",CommonUtils.encodePassword("")); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==",CommonUtils.encodePassword("123456")); + Assert.assertEquals("IUAjJCVeJipJVkZCV2xoVFYwQT0=",CommonUtils.encodePassword("!QAZXSW@")); + Assert.assertEquals("IUAjJCVeJipOV1JtWjJWeUtFQT0=",CommonUtils.encodePassword("5dfger(@")); + + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false"); + + Assert.assertEquals("",CommonUtils.encodePassword("")); + Assert.assertEquals("123456",CommonUtils.encodePassword("123456")); + Assert.assertEquals("!QAZXSW@",CommonUtils.encodePassword("!QAZXSW@")); + Assert.assertEquals("5dfger(@",CommonUtils.encodePassword("5dfger(@")); + + } + + @Test + public void decodePassword() { + + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true"); + + Assert.assertEquals("", CommonUtils.decodePassword("")); + Assert.assertEquals("123456", CommonUtils.decodePassword("IUAjJCVeJipNVEl6TkRVMg==")); + Assert.assertEquals("!QAZXSW@", CommonUtils.decodePassword("IUAjJCVeJipJVkZCV2xoVFYwQT0=")); + Assert.assertEquals("5dfger(@", CommonUtils.decodePassword("IUAjJCVeJipOV1JtWjJWeUtFQT0=")); + + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false"); + + Assert.assertEquals("", CommonUtils.decodePassword("")); + Assert.assertEquals("123456", CommonUtils.decodePassword("123456")); + Assert.assertEquals("!QAZXSW@", CommonUtils.decodePassword("!QAZXSW@")); + Assert.assertEquals("5dfger(@", CommonUtils.decodePassword("5dfger(@")); + } + } \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java index 96217842bf..b2a255b2e2 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; +import org.apache.dolphinscheduler.common.Constants; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -67,4 +68,17 @@ public class FileUtilsTest { Assert.assertTrue(false); } } + + @Test + public void testSetValue() { + try { + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true"); + Assert.assertTrue(PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE)); + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false"); + Assert.assertFalse(PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE)); + } catch (Exception e) { + Assert.assertTrue(false); + } + } + } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java index a8cad85b45..ccae36aae0 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java @@ -20,6 +20,7 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -182,8 +183,12 @@ public abstract class BaseDataSource { this.user = user; } + /** + * password need decode + * @return + */ public String getPassword() { - return password; + return CommonUtils.decodePassword(password); } public void setPassword(String password) { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java index 50e5e7b996..9fb756d3a7 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java @@ -82,6 +82,8 @@ public class MySQLDataSource extends BaseDataSource { @Override public String getPassword() { + // password need decode + password = super.getPassword(); if(password.contains(sensitiveParam)){ logger.warn("sensitive param : {} in password field is filtered", sensitiveParam); password = password.replace(sensitiveParam, ""); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java index 6c44c3e329..9250e500d1 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.junit.Assert; import org.junit.Test; @@ -112,4 +114,51 @@ public class BaseDataSourceTest { } + + @Test + public void testGetPassword() { + BaseDataSource dataSource = new BaseDataSource() { + @Override + public String driverClassSelector() { + return null; + } + + @Override + public DbType dbTypeSelector() { + return null; + } + }; + + String password= ""; + dataSource.setPassword(password); + Assert.assertEquals("", dataSource.getPassword()); + password= "IUAjJCVeJipNVEl6TkRVMg=="; + dataSource.setPassword(password); + Assert.assertNotNull(dataSource.getPassword()); + Assert.assertNotNull(dataSource.getPassword()); + + dataSource.setPassword(password); + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true"); + Assert.assertEquals("123456", dataSource.getPassword()); + + dataSource.setPassword(password); + Assert.assertEquals("123456", dataSource.getPassword()); + Assert.assertEquals("123456", dataSource.getPassword()); + Assert.assertEquals("123456", dataSource.getPassword()); + + dataSource.setPassword(password); + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false"); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + + dataSource.setPassword(password); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + + + } + + + + } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSourceTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSourceTest.java index 59bf5c412e..2e9e904cd9 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSourceTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSourceTest.java @@ -16,6 +16,8 @@ */ package org.apache.dolphinscheduler.dao.datasource; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.junit.Assert; import org.junit.Test; @@ -46,6 +48,7 @@ public class MySQLDataSourceTest { Assert.assertEquals("test_pwd?", dataSource.getPassword()); } + @Test public void testFilterOther(){ MySQLDataSource dataSource = new MySQLDataSource(); @@ -61,4 +64,36 @@ public class MySQLDataSourceTest { other = dataSource.filterOther("serverTimezone=Asia/Shanghai&autoDeserialize=true&characterEncoding=utf8"); Assert.assertEquals("serverTimezone=Asia/Shanghai&characterEncoding=utf8", other); } + + @Test + public void testGetPasswordWithDecodePassword(){ + MySQLDataSource dataSource = new MySQLDataSource(); + String password= ""; + dataSource.setPassword(password); + Assert.assertEquals("", dataSource.getPassword()); + password= "IUAjJCVeJipNVEl6TkRVMg=="; + dataSource.setPassword(password); + Assert.assertNotNull(dataSource.getPassword()); + Assert.assertNotNull(dataSource.getPassword()); + + dataSource.setPassword(password); + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true"); + Assert.assertEquals("123456", dataSource.getPassword()); + + dataSource.setPassword(password); + Assert.assertEquals("123456", dataSource.getPassword()); + Assert.assertEquals("123456", dataSource.getPassword()); + Assert.assertEquals("123456", dataSource.getPassword()); + + dataSource.setPassword(password); + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false"); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + + dataSource.setPassword(password); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + + } + } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProcessInstanceLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProcessInstanceLocator.java index 1af9d56274..445ea3ee78 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProcessInstanceLocator.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProcessInstanceLocator.java @@ -21,7 +21,7 @@ import org.openqa.selenium.By; public class ProcessInstanceLocator { // jump Process Instance page //click Process Instance name - public static final By CLICK_PROCESS_INSTANCE_NAME = By.xpath("//div[3]/div/ul/li[2]"); + public static final By CLICK_PROCESS_INSTANCE_NAME = By.xpath("//div[4]/div/ul/li[2]"); // click rerun button public static final By CLICK_RERUN_BUTTON = By.xpath("//tr[2]/td[14]/div[1]/button[2]");