github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Merge branch 'dev' of https://github.com/apache/incubator-dolphinscheduler into dev

pull/3/MERGE
break60 4 years ago
parent
cab72aeac0 78eb07b361
commit
7ca60c7e61
4 changed files with 268 additions and 104 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 10
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java
  2. 140
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java
  3. 220
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java
  4. 2
      pom.xml

10
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java
Unescape View File

@ -100,7 +100,7 @@ public class DataSourceController extends BaseController {
@RequestParam(value = "other") String other) { @RequestParam(value = "other") String other) {
logger.info("login user {} create datasource name: {}, note: {}, type: {}, host: {}, port: {}, database : {}, principal: {}, userName : {}, connectType: {}, other: {}", logger.info("login user {} create datasource name: {}, note: {}, type: {}, host: {}, port: {}, database : {}, principal: {}, userName : {}, connectType: {}, other: {}",
loginUser.getUserName(), name, note, type, host, port, database, principal, userName, connectType, other); loginUser.getUserName(), name, note, type, host, port, database, principal, userName, connectType, other);
String parameter = dataSourceService.buildParameter(name, note, type, host, port, database, principal, userName, password, connectType, other); String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other);
Map<String, Object> result = dataSourceService.createDataSource(loginUser, name, note, type, parameter); Map<String, Object> result = dataSourceService.createDataSource(loginUser, name, note, type, parameter);
return returnDataList(result); return returnDataList(result);
} }
@ -155,7 +155,7 @@ public class DataSourceController extends BaseController {
@RequestParam(value = "other") String other) { @RequestParam(value = "other") String other) {
logger.info("login user {} updateProcessInstance datasource name: {}, note: {}, type: {}, connectType: {}, other: {}", logger.info("login user {} updateProcessInstance datasource name: {}, note: {}, type: {}, connectType: {}, other: {}",
loginUser.getUserName(), name, note, type, connectType, other); loginUser.getUserName(), name, note, type, connectType, other);
String parameter = dataSourceService.buildParameter(name, note, type, host, port, database, principal, userName, password, connectType, other); String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other);
Map<String, Object> dataSource = dataSourceService.updateDataSource(id, loginUser, name, note, type, parameter); Map<String, Object> dataSource = dataSourceService.updateDataSource(id, loginUser, name, note, type, parameter);
return returnDataList(dataSource); return returnDataList(dataSource);
} }
@ -280,7 +280,7 @@ public class DataSourceController extends BaseController {
@RequestParam(value = "other") String other) { @RequestParam(value = "other") String other) {
logger.info("login user {}, connect datasource: {}, note: {}, type: {}, connectType: {}, other: {}", logger.info("login user {}, connect datasource: {}, note: {}, type: {}, connectType: {}, other: {}",
loginUser.getUserName(), name, note, type, connectType, other); loginUser.getUserName(), name, note, type, connectType, other);
String parameter = dataSourceService.buildParameter(name, note, type, host, port, database, principal, userName, password, connectType, other); String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other);
Boolean isConnection = dataSourceService.checkConnection(type, parameter); Boolean isConnection = dataSourceService.checkConnection(type, parameter);
Result result = new Result(); Result result = new Result();
@ -310,7 +310,7 @@ public class DataSourceController extends BaseController {
@RequestParam("id") int id) { @RequestParam("id") int id) {
logger.info("connection test, login user:{}, id:{}", loginUser.getUserName(), id); logger.info("connection test, login user:{}, id:{}", loginUser.getUserName(), id);
Boolean isConnection = dataSourceService.connectionTest(loginUser, id); Boolean isConnection = dataSourceService.connectionTest(id);
Result result = new Result(); Result result = new Result();
if (isConnection) { if (isConnection) {
@ -361,7 +361,7 @@ public class DataSourceController extends BaseController {
logger.info("login user {}, verfiy datasource name: {}", logger.info("login user {}, verfiy datasource name: {}",
loginUser.getUserName(), name); loginUser.getUserName(), name);
return dataSourceService.verifyDataSourceName(loginUser, name); return dataSourceService.verifyDataSourceName(name);
} }

140
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java
Unescape View File

@ -53,7 +53,7 @@ import static org.apache.dolphinscheduler.common.utils.PropertyUtils.getString;
* datasource service * datasource service
*/ */
@Service @Service
public class DataSourceService extends BaseService{ public class DataSourceService extends BaseService {
private static final Logger logger = LoggerFactory.getLogger(DataSourceService.class); private static final Logger logger = LoggerFactory.getLogger(DataSourceService.class);
@ -65,7 +65,6 @@ public class DataSourceService extends BaseService{
public static final String PRINCIPAL = "principal"; public static final String PRINCIPAL = "principal";
public static final String DATABASE = "database"; public static final String DATABASE = "database";
public static final String USER_NAME = "userName"; public static final String USER_NAME = "userName";
public static final String PASSWORD = Constants.PASSWORD;
public static final String OTHER = "other"; public static final String OTHER = "other";
@ -80,9 +79,9 @@ public class DataSourceService extends BaseService{
* create data source * create data source
* *
* @param loginUser login user * @param loginUser login user
* @param name data source name * @param name data source name
* @param desc data source description * @param desc data source description
* @param type data source type * @param type data source type
* @param parameter datasource parameters * @param parameter datasource parameters
* @return create result code * @return create result code
*/ */
@ -131,11 +130,11 @@ public class DataSourceService extends BaseService{
* updateProcessInstance datasource * updateProcessInstance datasource
* *
* @param loginUser login user * @param loginUser login user
* @param name data source name * @param name data source name
* @param desc data source description * @param desc data source description
* @param type data source type * @param type data source type
* @param parameter datasource parameters * @param parameter datasource parameters
* @param id data source id * @param id data source id
* @return update result code * @return update result code
*/ */
public Map<String, Object> updateDataSource(int id, User loginUser, String name, String desc, DbType type, String parameter) { public Map<String, Object> updateDataSource(int id, User loginUser, String name, String desc, DbType type, String parameter) {
@ -148,13 +147,13 @@ public class DataSourceService extends BaseService{
return result; return result;
} }
if(!hasPerm(loginUser, dataSource.getUserId())){ if (!hasPerm(loginUser, dataSource.getUserId())) {
putMsg(result, Status.USER_NO_OPERATION_PERM); putMsg(result, Status.USER_NO_OPERATION_PERM);
return result; return result;
} }
//check name can use or not //check name can use or not
if(!name.trim().equals(dataSource.getName()) && checkName(name)){ if (!name.trim().equals(dataSource.getName()) && checkName(name)) {
putMsg(result, Status.DATASOURCE_EXIST); putMsg(result, Status.DATASOURCE_EXIST);
return result; return result;
} }
@ -199,6 +198,7 @@ public class DataSourceService extends BaseService{
/** /**
* updateProcessInstance datasource * updateProcessInstance datasource
*
* @param id datasource id * @param id datasource id
* @return data source detail * @return data source detail
*/ */
@ -220,11 +220,11 @@ public class DataSourceService extends BaseService{
String parameter = dataSource.getConnectionParams(); String parameter = dataSource.getConnectionParams();
BaseDataSource datasourceForm = DataSourceFactory.getDatasource(dataSource.getType(), parameter); BaseDataSource datasourceForm = DataSourceFactory.getDatasource(dataSource.getType(), parameter);
DbConnectType connectType = null; DbConnectType connectType = null;
String hostSeperator = Constants.DOUBLE_SLASH; String hostSeperator = Constants.DOUBLE_SLASH;
if(DbType.ORACLE.equals(dataSource.getType())){ if (DbType.ORACLE.equals(dataSource.getType())) {
connectType = ((OracleDataSource) datasourceForm).getConnectType(); connectType = ((OracleDataSource) datasourceForm).getConnectType();
if(DbConnectType.ORACLE_SID.equals(connectType)){ if (DbConnectType.ORACLE_SID.equals(connectType)) {
hostSeperator = Constants.AT_SIGN; hostSeperator = Constants.AT_SIGN;
} }
} }
@ -233,7 +233,7 @@ public class DataSourceService extends BaseService{
String other = datasourceForm.getOther(); String other = datasourceForm.getOther();
String address = datasourceForm.getAddress(); String address = datasourceForm.getAddress();
String[] hostsPorts = getHostsAndPort(address,hostSeperator); String[] hostsPorts = getHostsAndPort(address, hostSeperator);
// ip host // ip host
String host = hostsPorts[0]; String host = hostsPorts[0];
// prot // prot
@ -285,14 +285,13 @@ public class DataSourceService extends BaseService{
return result; return result;
} }
/** /**
* query datasource list by keyword * query datasource list by keyword
* *
* @param loginUser login user * @param loginUser login user
* @param searchVal search value * @param searchVal search value
* @param pageNo page number * @param pageNo page number
* @param pageSize page size * @param pageSize page size
* @return data source list page * @return data source list page
*/ */
public Map<String, Object> queryDataSourceListPaging(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { public Map<String, Object> queryDataSourceListPaging(User loginUser, String searchVal, Integer pageNo, Integer pageSize) {
@ -302,14 +301,14 @@ public class DataSourceService extends BaseService{
if (isAdmin(loginUser)) { if (isAdmin(loginUser)) {
dataSourceList = dataSourceMapper.selectPaging(dataSourcePage, 0, searchVal); dataSourceList = dataSourceMapper.selectPaging(dataSourcePage, 0, searchVal);
}else{ } else {
dataSourceList = dataSourceMapper.selectPaging(dataSourcePage, loginUser.getId(), searchVal); dataSourceList = dataSourceMapper.selectPaging(dataSourcePage, loginUser.getId(), searchVal);
} }
List<DataSource> dataSources = dataSourceList.getRecords(); List<DataSource> dataSources = dataSourceList != null ? dataSourceList.getRecords() : new ArrayList<>();
handlePasswd(dataSources); handlePasswd(dataSources);
PageInfo pageInfo = new PageInfo<Resource>(pageNo, pageSize); PageInfo pageInfo = new PageInfo<Resource>(pageNo, pageSize);
pageInfo.setTotalCount((int)(dataSourceList.getTotal())); pageInfo.setTotalCount((int) (dataSourceList != null ? dataSourceList.getTotal() : 0L));
pageInfo.setLists(dataSources); pageInfo.setLists(dataSources);
result.put(Constants.DATA_LIST, pageInfo); result.put(Constants.DATA_LIST, pageInfo);
putMsg(result, Status.SUCCESS); putMsg(result, Status.SUCCESS);
@ -319,14 +318,15 @@ public class DataSourceService extends BaseService{
/** /**
* handle datasource connection password for safety * handle datasource connection password for safety
*
* @param dataSourceList * @param dataSourceList
*/ */
private void handlePasswd(List<DataSource> dataSourceList) { private void handlePasswd(List<DataSource> dataSourceList) {
for (DataSource dataSource : dataSourceList) { for (DataSource dataSource : dataSourceList) {
String connectionParams = dataSource.getConnectionParams(); String connectionParams = dataSource.getConnectionParams();
ObjectNode object = JSONUtils.parseObject(connectionParams); ObjectNode object = JSONUtils.parseObject(connectionParams);
object.put(Constants.PASSWORD, Constants.XXXXXX); object.put(Constants.PASSWORD, Constants.XXXXXX);
dataSource.setConnectionParams(object.toString()); dataSource.setConnectionParams(object.toString());
@ -337,7 +337,7 @@ public class DataSourceService extends BaseService{
* query data resource list * query data resource list
* *
* @param loginUser login user * @param loginUser login user
* @param type data source type * @param type data source type
* @return data source list page * @return data source list page
*/ */
public Map<String, Object> queryDataSourceList(User loginUser, Integer type) { public Map<String, Object> queryDataSourceList(User loginUser, Integer type) {
@ -347,7 +347,7 @@ public class DataSourceService extends BaseService{
if (isAdmin(loginUser)) { if (isAdmin(loginUser)) {
datasourceList = dataSourceMapper.listAllDataSourceByType(type); datasourceList = dataSourceMapper.listAllDataSourceByType(type);
}else{ } else {
datasourceList = dataSourceMapper.queryDataSourceByType(loginUser.getId(), type); datasourceList = dataSourceMapper.queryDataSourceByType(loginUser.getId(), type);
} }
@ -360,11 +360,10 @@ public class DataSourceService extends BaseService{
/** /**
* verify datasource exists * verify datasource exists
* *
* @param loginUser login user * @param name datasource name
* @param name datasource name
* @return true if data datasource not exists, otherwise return false * @return true if data datasource not exists, otherwise return false
*/ */
public Result verifyDataSourceName(User loginUser, String name) { public Result verifyDataSourceName(String name) {
Result result = new Result(); Result result = new Result();
List<DataSource> dataSourceList = dataSourceMapper.queryDataSourceByName(name); List<DataSource> dataSourceList = dataSourceMapper.queryDataSourceByName(name);
if (dataSourceList != null && dataSourceList.size() > 0) { if (dataSourceList != null && dataSourceList.size() > 0) {
@ -380,7 +379,7 @@ public class DataSourceService extends BaseService{
/** /**
* get connection * get connection
* *
* @param dbType datasource type * @param dbType datasource type
* @param parameter parameter * @param parameter parameter
* @return connection for datasource * @return connection for datasource
*/ */
@ -399,18 +398,18 @@ public class DataSourceService extends BaseService{
break; break;
case HIVE: case HIVE:
case SPARK: case SPARK:
if (CommonUtils.getKerberosStartupState()) { if (CommonUtils.getKerberosStartupState()) {
System.setProperty(org.apache.dolphinscheduler.common.Constants.JAVA_SECURITY_KRB5_CONF, System.setProperty(org.apache.dolphinscheduler.common.Constants.JAVA_SECURITY_KRB5_CONF,
getString(org.apache.dolphinscheduler.common.Constants.JAVA_SECURITY_KRB5_CONF_PATH)); getString(org.apache.dolphinscheduler.common.Constants.JAVA_SECURITY_KRB5_CONF_PATH));
Configuration configuration = new Configuration(); Configuration configuration = new Configuration();
configuration.set(org.apache.dolphinscheduler.common.Constants.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); configuration.set(org.apache.dolphinscheduler.common.Constants.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
UserGroupInformation.setConfiguration(configuration); UserGroupInformation.setConfiguration(configuration);
UserGroupInformation.loginUserFromKeytab(getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_USERNAME), UserGroupInformation.loginUserFromKeytab(getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_USERNAME),
getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_PATH)); getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_PATH));
} }
if (dbType == DbType.HIVE){ if (dbType == DbType.HIVE) {
datasource = JSONUtils.parseObject(parameter, HiveDataSource.class); datasource = JSONUtils.parseObject(parameter, HiveDataSource.class);
}else if (dbType == DbType.SPARK){ } else if (dbType == DbType.SPARK) {
datasource = JSONUtils.parseObject(parameter, SparkDataSource.class); datasource = JSONUtils.parseObject(parameter, SparkDataSource.class);
} }
Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER); Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER);
@ -439,20 +438,19 @@ public class DataSourceService extends BaseService{
break; break;
} }
if(datasource != null){ if (datasource != null) {
connection = DriverManager.getConnection(datasource.getJdbcUrl(), datasource.getUser(), datasource.getPassword()); connection = DriverManager.getConnection(datasource.getJdbcUrl(), datasource.getUser(), datasource.getPassword());
} }
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(),e); logger.error(e.getMessage(), e);
} }
return connection; return connection;
} }
/** /**
* check connection * check connection
* *
* @param type data source type * @param type data source type
* @param parameter data source parameters * @param parameter data source parameters
* @return true if connect successfully, otherwise false * @return true if connect successfully, otherwise false
*/ */
@ -470,35 +468,35 @@ public class DataSourceService extends BaseService{
return isConnection; return isConnection;
} }
/** /**
* test connection * test connection
* *
* @param loginUser login user
* @param id datasource id * @param id datasource id
* @return connect result code * @return connect result code
*/ */
public boolean connectionTest(User loginUser, int id) { public boolean connectionTest(int id) {
DataSource dataSource = dataSourceMapper.selectById(id); DataSource dataSource = dataSourceMapper.selectById(id);
return checkConnection(dataSource.getType(), dataSource.getConnectionParams()); if (dataSource != null) {
return checkConnection(dataSource.getType(), dataSource.getConnectionParams());
} else {
return false;
}
} }
/** /**
* build paramters * build paramters
* *
* @param name data source name * @param type data source type
* @param desc data source description * @param host data source host
* @param type data source type * @param port data source port
* @param host data source host * @param database data source database name
* @param port data source port * @param userName user name
* @param database data source database name * @param password password
* @param userName user name * @param other other parameters
* @param password password
* @param other other parameters
* @param principal principal * @param principal principal
* @return datasource parameter * @return datasource parameter
*/ */
public String buildParameter(String name, String desc, DbType type, String host, public String buildParameter(DbType type, String host,
String port, String database, String principal, String userName, String port, String database, String principal, String userName,
String password, DbConnectType connectType, String other) { String password, DbConnectType connectType, String other) {
@ -510,7 +508,7 @@ public class DataSourceService extends BaseService{
} }
if (CommonUtils.getKerberosStartupState() && if (CommonUtils.getKerberosStartupState() &&
(type == DbType.HIVE || type == DbType.SPARK)){ (type == DbType.HIVE || type == DbType.SPARK)) {
jdbcUrl += ";principal=" + principal; jdbcUrl += ";principal=" + principal;
} }
@ -535,14 +533,14 @@ public class DataSourceService extends BaseService{
parameterMap.put(Constants.USER, userName); parameterMap.put(Constants.USER, userName);
parameterMap.put(Constants.PASSWORD, CommonUtils.encodePassword(password)); parameterMap.put(Constants.PASSWORD, CommonUtils.encodePassword(password));
if (CommonUtils.getKerberosStartupState() && if (CommonUtils.getKerberosStartupState() &&
(type == DbType.HIVE || type == DbType.SPARK)){ (type == DbType.HIVE || type == DbType.SPARK)) {
parameterMap.put(Constants.PRINCIPAL,principal); parameterMap.put(Constants.PRINCIPAL, principal);
} }
if (other != null && !"".equals(other)) { if (other != null && !"".equals(other)) {
Map<String, String> map = JSONUtils.toMap(other); Map<String, String> map = JSONUtils.toMap(other);
if (map.size() > 0) { if (map.size() > 0) {
StringBuilder otherSb = new StringBuilder(); StringBuilder otherSb = new StringBuilder();
for (Map.Entry<String, String> entry: map.entrySet()) { for (Map.Entry<String, String> entry : map.entrySet()) {
otherSb.append(String.format("%s=%s%s", entry.getKey(), entry.getValue(), separator)); otherSb.append(String.format("%s=%s%s", entry.getKey(), entry.getValue(), separator));
} }
if (!Constants.DB2.equals(type.name())) { if (!Constants.DB2.equals(type.name())) {
@ -553,7 +551,7 @@ public class DataSourceService extends BaseService{
} }
if(logger.isDebugEnabled()){ if (logger.isDebugEnabled()) {
logger.info("parameters map:{}", JSONUtils.toJsonString(parameterMap)); logger.info("parameters map:{}", JSONUtils.toJsonString(parameterMap));
} }
return JSONUtils.toJsonString(parameterMap); return JSONUtils.toJsonString(parameterMap);
@ -605,7 +603,7 @@ public class DataSourceService extends BaseService{
/** /**
* delete datasource * delete datasource
* *
* @param loginUser login user * @param loginUser login user
* @param datasourceId data source id * @param datasourceId data source id
* @return delete result code * @return delete result code
*/ */
@ -615,12 +613,12 @@ public class DataSourceService extends BaseService{
try { try {
//query datasource by id //query datasource by id
DataSource dataSource = dataSourceMapper.selectById(datasourceId); DataSource dataSource = dataSourceMapper.selectById(datasourceId);
if(dataSource == null){ if (dataSource == null) {
logger.error("resource id {} not exist", datasourceId); logger.error("resource id {} not exist", datasourceId);
putMsg(result, Status.RESOURCE_NOT_EXIST); putMsg(result, Status.RESOURCE_NOT_EXIST);
return result; return result;
} }
if(!hasPerm(loginUser, dataSource.getUserId())){ if (!hasPerm(loginUser, dataSource.getUserId())) {
putMsg(result, Status.USER_NO_OPERATION_PERM); putMsg(result, Status.USER_NO_OPERATION_PERM);
return result; return result;
} }
@ -628,7 +626,7 @@ public class DataSourceService extends BaseService{
datasourceUserMapper.deleteByDatasourceId(datasourceId); datasourceUserMapper.deleteByDatasourceId(datasourceId);
putMsg(result, Status.SUCCESS); putMsg(result, Status.SUCCESS);
} catch (Exception e) { } catch (Exception e) {
logger.error("delete datasource error",e); logger.error("delete datasource error", e);
throw new RuntimeException("delete datasource error"); throw new RuntimeException("delete datasource error");
} }
return result; return result;
@ -638,7 +636,7 @@ public class DataSourceService extends BaseService{
* unauthorized datasource * unauthorized datasource
* *
* @param loginUser login user * @param loginUser login user
* @param userId user id * @param userId user id
* @return unauthed data source result code * @return unauthed data source result code
*/ */
public Map<String, Object> unauthDatasource(User loginUser, Integer userId) { public Map<String, Object> unauthDatasource(User loginUser, Integer userId) {
@ -679,7 +677,7 @@ public class DataSourceService extends BaseService{
* authorized datasource * authorized datasource
* *
* @param loginUser login user * @param loginUser login user
* @param userId user id * @param userId user id
* @return authorized result code * @return authorized result code
*/ */
public Map<String, Object> authedDatasource(User loginUser, Integer userId) { public Map<String, Object> authedDatasource(User loginUser, Integer userId) {
@ -700,11 +698,11 @@ public class DataSourceService extends BaseService{
/** /**
* get host and port by address * get host and port by address
* *
* @param address address * @param address address
* @return sting array: [host,port] * @return sting array: [host,port]
*/ */
private String[] getHostsAndPort(String address) { private String[] getHostsAndPort(String address) {
return getHostsAndPort(address,Constants.DOUBLE_SLASH); return getHostsAndPort(address, Constants.DOUBLE_SLASH);
} }
/** /**
@ -714,7 +712,7 @@ public class DataSourceService extends BaseService{
* @param separator separator * @param separator separator
* @return sting array: [host,port] * @return sting array: [host,port]
*/ */
private String[] getHostsAndPort(String address,String separator) { private String[] getHostsAndPort(String address, String separator) {
String[] result = new String[2]; String[] result = new String[2];
String[] tmpArray = address.split(separator); String[] tmpArray = address.split(separator);
String hostsAndPorts = tmpArray[tmpArray.length - 1]; String hostsAndPorts = tmpArray[tmpArray.length - 1];

220
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java
Unescape View File

@ -22,10 +22,14 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbConnectType; import org.apache.dolphinscheduler.common.enums.DbConnectType;
import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
import org.apache.dolphinscheduler.dao.datasource.MySQLDataSource;
import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
@ -35,8 +39,6 @@ import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito; import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.modules.junit4.PowerMockRunner; import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -45,16 +47,172 @@ import java.util.Map;
@RunWith(PowerMockRunner.class) @RunWith(PowerMockRunner.class)
@PowerMockIgnore({"sun.security.*", "javax.net.*"}) @PowerMockIgnore({"sun.security.*", "javax.net.*"})
public class DataSourceServiceTest { public class DataSourceServiceTest {
private static final Logger logger = LoggerFactory.getLogger(DataSourceServiceTest.class);
@InjectMocks @InjectMocks
private DataSourceService dataSourceService; private DataSourceService dataSourceService;
@Mock @Mock
private DataSourceMapper dataSourceMapper; private DataSourceMapper dataSourceMapper;
@Mock
private DataSourceUserMapper datasourceUserMapper;
public void createDataSourceTest() {
User loginUser = getAdminUser();
String dataSourceName = "dataSource01";
String dataSourceDesc = "test dataSource";
DbType dataSourceType = DbType.POSTGRESQL;
String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null);
// data source exits
List<DataSource> dataSourceList = new ArrayList<>();
DataSource dataSource = new DataSource();
dataSource.setName(dataSourceName);
dataSourceList.add(dataSource);
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(dataSourceList);
Map<String, Object> dataSourceExitsResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.DATASOURCE_EXIST, dataSourceExitsResult.get(Constants.STATUS));
// data source exits
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(false);
Map<String, Object> connectFailedResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED, connectFailedResult.get(Constants.STATUS));
// data source exits
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(true);
PowerMockito.when(DataSourceFactory.getDatasource(dataSourceType, parameter)).thenReturn(null);
Map<String, Object> notValidError = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, notValidError.get(Constants.STATUS));
// success
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(true);
PowerMockito.when(DataSourceFactory.getDatasource(dataSourceType, parameter)).thenReturn(JSONUtils.parseObject(parameter, MySQLDataSource.class));
Map<String, Object> success = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.SUCCESS, success.get(Constants.STATUS));
}
public void updateDataSourceTest() {
User loginUser = getAdminUser();
int dataSourceId = 12;
String dataSourceName = "dataSource01";
String dataSourceDesc = "test dataSource";
DbType dataSourceType = DbType.POSTGRESQL;
String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null);
// data source not exits
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(null);
Map<String, Object> resourceNotExits = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.RESOURCE_NOT_EXIST, resourceNotExits.get(Constants.STATUS));
// user no operation perm
DataSource dataSource = new DataSource();
dataSource.setUserId(0);
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource);
Map<String, Object> userNoOperationPerm = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, userNoOperationPerm.get(Constants.STATUS));
// data source name exits
dataSource.setUserId(-1);
List<DataSource> dataSourceList = new ArrayList<>();
dataSourceList.add(dataSource);
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource);
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(dataSourceList);
Map<String, Object> dataSourceNameExist = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.DATASOURCE_EXIST, dataSourceNameExist.get(Constants.STATUS));
// data source connect failed
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource);
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(null);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(true);
Map<String, Object> connectFailed = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED, connectFailed.get(Constants.STATUS));
//success
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource);
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(null);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(false);
Map<String, Object> success = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.SUCCESS, connectFailed.get(Constants.STATUS));
}
@Test
public void queryDataSourceListPagingTest() {
User loginUser = getAdminUser();
String searchVal = "";
int pageNo = 1;
int pageSize = 10;
Map<String, Object> success = dataSourceService.queryDataSourceListPaging(loginUser, searchVal, pageNo, pageSize);
Assert.assertEquals(Status.SUCCESS, success.get(Constants.STATUS));
}
@Test
public void connectionTest() {
int dataSourceId = -1;
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(null);
Assert.assertFalse(dataSourceService.connectionTest(dataSourceId));
}
@Test @Test
public void queryDataSourceListTest(){ public void deleteTest() {
User loginUser = getAdminUser();
int dataSourceId = 1;
Result result = new Result();
//resource not exist
dataSourceService.putMsg(result, Status.RESOURCE_NOT_EXIST);
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(null);
Assert.assertEquals(result.getCode(), dataSourceService.delete(loginUser, dataSourceId).getCode());
// user no operation perm
dataSourceService.putMsg(result, Status.USER_NO_OPERATION_PERM);
DataSource dataSource = new DataSource();
dataSource.setUserId(0);
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource);
Assert.assertEquals(result.getCode(), dataSourceService.delete(loginUser, dataSourceId).getCode());
// success
dataSourceService.putMsg(result, Status.SUCCESS);
dataSource.setUserId(-1);
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource);
Assert.assertEquals(result.getCode(), dataSourceService.delete(loginUser, dataSourceId).getCode());
}
@Test
public void unauthDatasourceTest() {
User loginUser = getAdminUser();
int userId = -1;
//user no operation perm
Map<String, Object> noOperationPerm = dataSourceService.unauthDatasource(loginUser, userId);
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, noOperationPerm.get(Constants.STATUS));
//success
loginUser.setUserType(UserType.ADMIN_USER);
Map<String, Object> success = dataSourceService.unauthDatasource(loginUser, userId);
Assert.assertEquals(Status.SUCCESS, success.get(Constants.STATUS));
}
@Test
public void authedDatasourceTest() {
User loginUser = getAdminUser();
int userId = -1;
//user no operation perm
Map<String, Object> noOperationPerm = dataSourceService.authedDatasource(loginUser, userId);
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, noOperationPerm.get(Constants.STATUS));
//success
loginUser.setUserType(UserType.ADMIN_USER);
Map<String, Object> success = dataSourceService.authedDatasource(loginUser, userId);
Assert.assertEquals(Status.SUCCESS, success.get(Constants.STATUS));
}
@Test
public void queryDataSourceListTest() {
User loginUser = new User(); User loginUser = new User();
loginUser.setUserType(UserType.GENERAL_USER); loginUser.setUserType(UserType.GENERAL_USER);
Map<String, Object> map = dataSourceService.queryDataSourceList(loginUser, DbType.MYSQL.ordinal()); Map<String, Object> map = dataSourceService.queryDataSourceList(loginUser, DbType.MYSQL.ordinal());
@ -62,35 +220,34 @@ public class DataSourceServiceTest {
} }
@Test @Test
public void verifyDataSourceNameTest(){ public void verifyDataSourceNameTest() {
User loginUser = new User(); User loginUser = new User();
loginUser.setUserType(UserType.GENERAL_USER); loginUser.setUserType(UserType.GENERAL_USER);
String dataSourceName = "dataSource1"; String dataSourceName = "dataSource1";
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(getDataSourceList()); PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(getDataSourceList());
Result result = dataSourceService.verifyDataSourceName(loginUser, dataSourceName); Result result = dataSourceService.verifyDataSourceName(dataSourceName);
Assert.assertEquals(Status.DATASOURCE_EXIST.getMsg(),result.getMsg()); Assert.assertEquals(Status.DATASOURCE_EXIST.getMsg(), result.getMsg());
} }
@Test @Test
public void queryDataSourceTest(){ public void queryDataSourceTest() {
PowerMockito.when(dataSourceMapper.selectById(Mockito.anyInt())).thenReturn(null); PowerMockito.when(dataSourceMapper.selectById(Mockito.anyInt())).thenReturn(null);
Map<String, Object> result = dataSourceService.queryDataSource(Mockito.anyInt()); Map<String, Object> result = dataSourceService.queryDataSource(Mockito.anyInt());
Assert.assertEquals(((Status)result.get(Constants.STATUS)).getCode(),Status.RESOURCE_NOT_EXIST.getCode()); Assert.assertEquals(((Status) result.get(Constants.STATUS)).getCode(), Status.RESOURCE_NOT_EXIST.getCode());
PowerMockito.when(dataSourceMapper.selectById(Mockito.anyInt())).thenReturn(getOracleDataSource()); PowerMockito.when(dataSourceMapper.selectById(Mockito.anyInt())).thenReturn(getOracleDataSource());
result = dataSourceService.queryDataSource(Mockito.anyInt()); result = dataSourceService.queryDataSource(Mockito.anyInt());
Assert.assertEquals(((Status)result.get(Constants.STATUS)).getCode(),Status.SUCCESS.getCode()); Assert.assertEquals(((Status) result.get(Constants.STATUS)).getCode(), Status.SUCCESS.getCode());
} }
private List<DataSource> getDataSourceList() {
private List<DataSource> getDataSourceList(){ List<DataSource> dataSources = new ArrayList<>();
List<DataSource> dataSources = new ArrayList<>();
dataSources.add(getOracleDataSource()); dataSources.add(getOracleDataSource());
return dataSources; return dataSources;
} }
private DataSource getOracleDataSource(){ private DataSource getOracleDataSource() {
DataSource dataSource = new DataSource(); DataSource dataSource = new DataSource();
dataSource.setName("test"); dataSource.setName("test");
dataSource.setNote("Note"); dataSource.setNote("Note");
@ -101,31 +258,40 @@ public class DataSourceServiceTest {
} }
@Test @Test
public void buildParameter(){ public void buildParameter() {
String param = dataSourceService.buildParameter("","", DbType.ORACLE, "192.168.9.1","1521","im" String param = dataSourceService.buildParameter(DbType.ORACLE, "192.168.9.1", "1521", "im"
,"","test","test", DbConnectType.ORACLE_SERVICE_NAME,""); , "", "test", "test", DbConnectType.ORACLE_SERVICE_NAME, "");
String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}"; String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}";
Assert.assertEquals(expected, param); Assert.assertEquals(expected, param);
} }
@Test @Test
public void buildParameterWithDecodePassword(){ public void buildParameterWithDecodePassword() {
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true"); PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true");
String param = dataSourceService.buildParameter("name","desc", DbType.MYSQL, "192.168.9.1","1521","im" String param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im"
,"","test","123456", null,""); , "", "test", "123456", null, "");
String expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}"; String expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}";
Assert.assertEquals(expected, param); Assert.assertEquals(expected, param);
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false"); PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false");
param = dataSourceService.buildParameter("name","desc", DbType.MYSQL, "192.168.9.1","1521","im" param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im"
,"","test","123456", null,""); , "", "test", "123456", null, "");
expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"123456\"}"; expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"123456\"}";
Assert.assertEquals(expected, param); Assert.assertEquals(expected, param);
} }
/**
* get Mock Admin User
*
* @return admin user
*/
private User getAdminUser() {
User loginUser = new User();
loginUser.setId(-1);
loginUser.setUserName("admin");
loginUser.setUserType(UserType.GENERAL_USER);
return loginUser;
}
} }

2
pom.xml
Unescape View File

@ -724,7 +724,7 @@
<include>**/api/service/BaseDAGServiceTest.java</include> <include>**/api/service/BaseDAGServiceTest.java</include>
<include>**/api/service/BaseServiceTest.java</include> <include>**/api/service/BaseServiceTest.java</include>
<include>**/api/service/DataAnalysisServiceTest.java</include> <include>**/api/service/DataAnalysisServiceTest.java</include>
<!--<include>**/api/service/DataSourceServiceTest.java</include>--> <include>**/api/service/DataSourceServiceTest.java</include>
<include>**/api/service/ExecutorService2Test.java</include> <include>**/api/service/ExecutorService2Test.java</include>
<include>**/api/service/ExecutorServiceTest.java</include> <include>**/api/service/ExecutorServiceTest.java</include>
<include>**/api/service/LoggerServiceTest.java</include> <include>**/api/service/LoggerServiceTest.java</include>

Write Preview
Loading…
Cancel
Save