Browse Source

revert: Datasource need test binding to create prod (#14381)

We should keep our step as less as possible, for now, we have to
test datasource and binding it to the prod datasource, I think it will
make our users do not thing to use datasource, so I do like to revert
it and keep step as less as possible

related to https://github.com/apache/dolphinscheduler/pull/11670
3.2.1-prepare
Jay Chung 1 year ago committed by GitHub
parent
commit
221df4a278
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 12
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java
  2. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java
  3. 21
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java
  4. 2
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java
  5. 12
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java
  6. 19
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapper.java
  7. 16
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/DataSourceMapper.xml
  8. 2
      dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql
  9. 2
      dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql
  10. 2
      dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql
  11. 24
      dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl.sql
  12. 24
      dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_ddl.sql
  13. 4
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java
  14. 20
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseDataSourceParamDTO.java
  15. 2
      dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-starrocks/src/main/java/org/apache/dolphinscheduler/plugin/datasource/starrocks/param/StarRocksDataSourceParamDTO.java
  16. 4
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ClickhouseDataSourceE2ETest.java
  17. 3
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/HiveDataSourceE2ETest.java
  18. 3
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/MysqlDataSourceE2ETest.java
  19. 3
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/PostgresDataSourceE2ETest.java
  20. 3
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/SqlServerDataSourceE2ETest.java
  21. 16
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/datasource/DataSourcePage.java
  22. 2
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
  23. 8
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
  24. 15
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
  25. 8
      dolphinscheduler-ui/src/locales/en_US/datasource.ts
  26. 8
      dolphinscheduler-ui/src/locales/zh_CN/datasource.ts
  27. 2
      dolphinscheduler-ui/src/service/modules/data-source/types.ts
  28. 44
      dolphinscheduler-ui/src/utils/environmental-distinction.ts
  29. 39
      dolphinscheduler-ui/src/views/datasource/list/detail.tsx
  30. 10
      dolphinscheduler-ui/src/views/datasource/list/use-columns.ts
  31. 57
      dolphinscheduler-ui/src/views/datasource/list/use-form.ts
  32. 8
      dolphinscheduler-ui/src/views/projects/task/instance/use-table.ts
  33. 9
      dolphinscheduler-ui/src/views/projects/workflow/instance/use-table.ts

12
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java

@ -82,7 +82,7 @@ public class DataSourceController extends BaseController {
*
* @param loginUser login user
* @param jsonStr datasource param
* example: {"type":"MYSQL","name":"txx","note":"","host":"localhost","port":3306,"principal":"","javaSecurityKrb5Conf":"","loginUserKeytabUsername":"","loginUserKeytabPath":"","userName":"root","password":"xxx","database":"ds","connectType":"","other":{"serverTimezone":"GMT-8"},"id":2,"testFlag":0,"bindTestId":1}
* example: {"type":"MYSQL","name":"txx","note":"","host":"localhost","port":3306,"principal":"","javaSecurityKrb5Conf":"","loginUserKeytabUsername":"","loginUserKeytabPath":"","userName":"root","password":"xxx","database":"ds","connectType":"","other":{"serverTimezone":"GMT-8"},"id":2}
* @return create result code
*/
@Operation(summary = "createDataSource", description = "CREATE_DATA_SOURCE_NOTES")
@ -102,7 +102,7 @@ public class DataSourceController extends BaseController {
* @param loginUser login user
* @param id datasource id
* @param jsonStr datasource param
* example: {"type":"MYSQL","name":"txx","note":"","host":"localhost","port":3306,"principal":"","javaSecurityKrb5Conf":"","loginUserKeytabUsername":"","loginUserKeytabPath":"","userName":"root","password":"xxx","database":"ds","connectType":"","other":{"serverTimezone":"GMT-8"},"id":2,"testFlag":0,"bindTestId":1}
* example: {"type":"MYSQL","name":"txx","note":"","host":"localhost","port":3306,"principal":"","javaSecurityKrb5Conf":"","loginUserKeytabUsername":"","loginUserKeytabPath":"","userName":"root","password":"xxx","database":"ds","connectType":"","other":{"serverTimezone":"GMT-8"},"id":2}
* @return update result code
*/
@Operation(summary = "updateDataSource", description = "UPDATE_DATA_SOURCE_NOTES")
@ -146,7 +146,7 @@ public class DataSourceController extends BaseController {
}
/**
* query online/testDatasource by type
* query datasource by type
*
* @param loginUser login user
* @param type data source type
@ -155,16 +155,14 @@ public class DataSourceController extends BaseController {
@Operation(summary = "queryDataSourceList", description = "QUERY_DATA_SOURCE_LIST_BY_TYPE_NOTES")
@Parameters({
@Parameter(name = "type", description = "DB_TYPE", required = true, schema = @Schema(implementation = DbType.class)),
@Parameter(name = "testFlag", description = "DB_TEST_FLAG", required = true, schema = @Schema(implementation = int.class))
})
@GetMapping(value = "/list")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_DATASOURCE_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result queryDataSourceList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam("type") DbType type,
@RequestParam("testFlag") int testFlag) {
Map<String, Object> result = dataSourceService.queryDataSourceList(loginUser, type.ordinal(), testFlag);
@RequestParam("type") DbType type) {
Map<String, Object> result = dataSourceService.queryDataSourceList(loginUser, type.ordinal());
return returnDataList(result);
}

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java

@ -69,13 +69,13 @@ public interface DataSourceService {
Result queryDataSourceListPaging(User loginUser, String searchVal, Integer pageNo, Integer pageSize);
/**
* query online/test data resource list
* query data resource list
*
* @param loginUser login user
* @param type data source type
* @return data source list page
*/
Map<String, Object> queryDataSourceList(User loginUser, Integer type, int testFlag);
Map<String, Object> queryDataSourceList(User loginUser, Integer type);
/**
* verify datasource exists

21
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java

@ -133,8 +133,6 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
dataSource.setConnectionParams(JSONUtils.toJsonString(connectionParam));
dataSource.setCreateTime(now);
dataSource.setUpdateTime(now);
dataSource.setTestFlag(datasourceParam.getTestFlag());
dataSource.setBindTestId(datasourceParam.getBindTestId());
try {
dataSourceMapper.insert(dataSource);
putMsg(result, Status.SUCCESS);
@ -205,11 +203,6 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
dataSource.setType(dataSource.getType());
dataSource.setConnectionParams(JSONUtils.toJsonString(connectionParam));
dataSource.setUpdateTime(now);
if (dataSource.getTestFlag() == 1 && dataSourceParam.getTestFlag() == 0) {
clearBindTestId(id);
}
dataSource.setTestFlag(dataSourceParam.getTestFlag());
dataSource.setBindTestId(dataSourceParam.getBindTestId());
try {
dataSourceMapper.updateById(dataSource);
log.info("Update datasource complete, datasourceId:{}, datasourceName:{}.", dataSource.getId(),
@ -257,8 +250,7 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
baseDataSourceParamDTO.setId(dataSource.getId());
baseDataSourceParamDTO.setName(dataSource.getName());
baseDataSourceParamDTO.setNote(dataSource.getNote());
baseDataSourceParamDTO.setTestFlag(dataSource.getTestFlag());
baseDataSourceParamDTO.setBindTestId(dataSource.getBindTestId());
result.put(Constants.DATA_LIST, baseDataSourceParamDTO);
putMsg(result, Status.SUCCESS);
return result;
@ -330,12 +322,12 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
* @return data source list page
*/
@Override
public Map<String, Object> queryDataSourceList(User loginUser, Integer type, int testFlag) {
public Map<String, Object> queryDataSourceList(User loginUser, Integer type) {
Map<String, Object> result = new HashMap<>();
List<DataSource> datasourceList = null;
if (loginUser.getUserType().equals(UserType.ADMIN_USER)) {
datasourceList = dataSourceMapper.queryDataSourceByType(0, type, testFlag);
datasourceList = dataSourceMapper.queryDataSourceByType(0, type);
} else {
Set<Integer> ids = resourcePermissionCheckService
.userOwnedResourceIdsAcquisition(AuthorizationType.DATASOURCE, loginUser.getId(), log);
@ -345,8 +337,7 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
return result;
}
datasourceList = dataSourceMapper.selectBatchIds(ids).stream()
.filter(dataSource -> dataSource.getType().getCode() == type)
.filter(dataSource -> dataSource.getTestFlag() == testFlag).collect(Collectors.toList());
.filter(dataSource -> dataSource.getType().getCode() == type).collect(Collectors.toList());
}
result.put(Constants.DATA_LIST, datasourceList);
putMsg(result, Status.SUCCESS);
@ -460,7 +451,6 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
}
dataSourceMapper.deleteById(datasourceId);
datasourceUserMapper.deleteByDatasourceId(datasourceId);
clearBindTestId(datasourceId);
log.info("Delete datasource complete, datasourceId:{}.", datasourceId);
putMsg(result, Status.SUCCESS);
} catch (Exception e) {
@ -709,8 +699,5 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
}
}
}
private void clearBindTestId(Integer bindTestId) {
dataSourceMapper.clearBindTestId(bindTestId);
}
}

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java

@ -319,7 +319,7 @@ public class DataSourceServiceTest {
dataSource.setType(DbType.MYSQL);
Mockito.when(dataSourceMapper.selectBatchIds(dataSourceIds)).thenReturn(Collections.singletonList(dataSource));
Map<String, Object> map =
dataSourceService.queryDataSourceList(loginUser, DbType.MYSQL.ordinal(), Constants.TEST_FLAG_NO);
dataSourceService.queryDataSourceList(loginUser, DbType.MYSQL.ordinal());
Assertions.assertEquals(Status.SUCCESS, map.get(Constants.STATUS));
}

12
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java

@ -24,7 +24,6 @@ import java.util.Objects;
import lombok.Data;
import com.baomidou.mybatisplus.annotation.FieldFill;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
@ -81,17 +80,6 @@ public class DataSource {
*/
private Date updateTime;
/**
* test flag
*/
protected int testFlag;
/**
* bind test data source id
*/
@TableField(fill = FieldFill.INSERT_UPDATE)
protected Integer bindTestId;
@Override
public boolean equals(Object o) {
if (this == o) {

19
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapper.java

@ -33,13 +33,12 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
public interface DataSourceMapper extends BaseMapper<DataSource> {
/**
* query online/testDatasource by type
* query datasource by type
* @param userId userId
* @param type type
* @return datasource list
*/
List<DataSource> queryDataSourceByType(@Param("userId") int userId, @Param("type") Integer type,
@Param("testFlag") int testFlag);
List<DataSource> queryDataSourceByType(@Param("userId") int userId, @Param("type") Integer type);
/**
* datasource page
@ -110,18 +109,4 @@ public interface DataSourceMapper extends BaseMapper<DataSource> {
IPage<DataSource> selectPagingByIds(Page<DataSource> dataSourcePage,
@Param("dataSourceIds") List<Integer> dataSourceIds,
@Param("name") String name);
/**
* clearBindTestId
* @param bindTestId
* @return
*/
void clearBindTestId(@Param("bindTestId") Integer bindTestId);
/**
* queryTestDataSourceId
* @param onlineDataSourceId
* @return Integer
*/
Integer queryTestDataSourceId(@Param("dataSourceId") Integer onlineDataSourceId);
}

16
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/DataSourceMapper.xml

@ -26,7 +26,6 @@
<include refid="baseSql"/>
from t_ds_datasource
where type=#{type}
and test_flag=#{testFlag}
<if test="userId != 0">
and id in
(select datasource_id
@ -41,7 +40,7 @@
<select id="selectPaging" resultType="org.apache.dolphinscheduler.dao.entity.DataSource">
select
d.id, d.name, d.note, d.type, d.user_id, connection_params, d.create_time, d.update_time, d.test_flag, d.bind_test_id
d.id, d.name, d.note, d.type, d.user_id, connection_params, d.create_time, d.update_time
,
u.user_name as user_name
from t_ds_datasource d
@ -69,7 +68,7 @@
where name=#{name}
</select>
<select id="queryAuthedDatasource" resultType="org.apache.dolphinscheduler.dao.entity.DataSource">
select ds.id, ds.name, ds.note, ds.type, ds.user_id, ds.connection_params, ds.create_time, ds.update_time, ds.test_flag, ds.bind_test_id
select ds.id, ds.name, ds.note, ds.type, ds.user_id, ds.connection_params, ds.create_time, ds.update_time
from t_ds_datasource ds, t_ds_relation_datasource_user rel
where ds.id = rel.datasource_id AND rel.user_id = #{userId}
</select>
@ -116,7 +115,7 @@
<select id="selectPagingByIds" resultType="org.apache.dolphinscheduler.dao.entity.DataSource">
select
d.id, d.name, d.note, d.type, d.user_id, connection_params, d.create_time, d.update_time, d.test_flag, d.bind_test_id
d.id, d.name, d.note, d.type, d.user_id, connection_params, d.create_time, d.update_time
,
u.user_name as user_name
from t_ds_datasource d
@ -133,12 +132,5 @@
</if>
order by update_time desc
</select>
<select id="queryTestDataSourceId" resultType="java.lang.Integer">
select d.bind_test_id
from t_ds_datasource d
where d.id = #{dataSourceId}
</select>
<update id="clearBindTestId">
update t_ds_datasource d set bind_test_id=null where bind_test_id=#{bindTestId}
</update>
</mapper>

2
dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql

@ -361,8 +361,6 @@ CREATE TABLE t_ds_datasource
connection_params text NOT NULL,
create_time datetime NOT NULL,
update_time datetime DEFAULT NULL,
test_flag int DEFAULT NULL,
bind_test_id int DEFAULT NULL,
PRIMARY KEY (id),
UNIQUE KEY t_ds_datasource_name_un (name, type)
);

2
dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql

@ -366,8 +366,6 @@ CREATE TABLE `t_ds_datasource` (
`connection_params` text NOT NULL COMMENT 'json connection params',
`create_time` datetime NOT NULL COMMENT 'create time',
`update_time` datetime DEFAULT NULL COMMENT 'update time',
`test_flag` tinyint(4) DEFAULT NULL COMMENT 'test flag:0 normal, 1 testDataSource',
`bind_test_id` int(11) DEFAULT NULL COMMENT 'bind testDataSource id',
PRIMARY KEY (`id`),
UNIQUE KEY `t_ds_datasource_name_un` (`name`, `type`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE = utf8_bin;

2
dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql

@ -289,8 +289,6 @@ CREATE TABLE t_ds_datasource (
connection_params text NOT NULL ,
create_time timestamp NOT NULL ,
update_time timestamp DEFAULT NULL ,
test_flag int DEFAULT NULL ,
bind_test_id int DEFAULT NULL ,
PRIMARY KEY (id),
CONSTRAINT t_ds_datasource_name_un UNIQUE (name, type)
) ;

24
dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl.sql

@ -56,30 +56,6 @@ delimiter ;
CALL uc_dolphin_T_t_ds_error_command_R_test_flag;
DROP PROCEDURE uc_dolphin_T_t_ds_error_command_R_test_flag;
-- uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id
drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id;
delimiter d//
CREATE PROCEDURE uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id()
BEGIN
IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
WHERE TABLE_NAME='t_ds_datasource'
AND TABLE_SCHEMA=(SELECT DATABASE())
AND COLUMN_NAME ='test_flag')
and NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
WHERE TABLE_NAME='t_ds_datasource'
AND TABLE_SCHEMA=(SELECT DATABASE())
AND COLUMN_NAME ='bind_test_id')
THEN
ALTER TABLE t_ds_datasource ADD `test_flag` tinyint(4) DEFAULT null COMMENT 'test flag:0 normal, 1 testDataSource';
ALTER TABLE t_ds_datasource ADD `bind_test_id` int DEFAULT null COMMENT 'bind testDataSource id';
END IF;
END;
d//
delimiter ;
CALL uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id;
DROP PROCEDURE uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id;
-- uc_dolphin_T_t_ds_process_instance_R_test_flag
drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_process_instance_R_test_flag;

24
dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/postgresql/dolphinscheduler_ddl.sql

@ -58,30 +58,6 @@ delimiter ;
select uc_dolphin_T_t_ds_error_command_R_test_flag();
DROP FUNCTION uc_dolphin_T_t_ds_error_command_R_test_flag();
-- uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id
delimiter ;
DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id();
delimiter d//
CREATE FUNCTION uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id() RETURNS void AS $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
WHERE TABLE_CATALOG=current_database()
AND TABLE_SCHEMA=current_schema()
AND TABLE_NAME='t_ds_datasource'
AND COLUMN_NAME ='test_flag')
THEN
ALTER TABLE t_ds_datasource alter column test_flag type int;
ALTER TABLE t_ds_datasource alter column test_flag set DEFAULT NULL;
ALTER TABLE t_ds_datasource alter column bind_test_id type int;
ALTER TABLE t_ds_datasource alter column bind_test_id set DEFAULT NULL;
END IF;
END;
$$ LANGUAGE plpgsql;
d//
delimiter ;
select uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id();
DROP FUNCTION uc_dolphin_T_t_ds_datasource_R_test_flag_bind_test_id();
-- uc_dolphin_T_t_ds_process_instance_R_test_flag
delimiter ;
DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_process_instance_R_test_flag();

4
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java

@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.dao.mapper;
import static java.util.stream.Collectors.toList;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.dao.BaseDaoTest;
@ -126,7 +125,7 @@ public class DataSourceMapperTest extends BaseDaoTest {
Map<Integer, DataSource> datasourceMap = createDataSourceMap(userId, "test");
List<DataSource> actualDataSources = dataSourceMapper.queryDataSourceByType(
0, DbType.MYSQL.ordinal(), Constants.TEST_FLAG_NO);
0, DbType.MYSQL.ordinal());
Assertions.assertTrue(actualDataSources.size() >= 2);
@ -380,7 +379,6 @@ public class DataSourceMapperTest extends BaseDaoTest {
dataSource.setType(DbType.MYSQL);
dataSource.setNote("mysql test");
dataSource.setConnectionParams("hello mysql");
dataSource.setTestFlag(Constants.TEST_FLAG_NO);
dataSource.setUpdateTime(DateUtils.getCurrentDate());
dataSource.setCreateTime(DateUtils.getCurrentDate());

20
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseDataSourceParamDTO.java

@ -45,10 +45,6 @@ public abstract class BaseDataSourceParamDTO implements Serializable {
protected String password;
protected int testFlag;
protected Integer bindTestId;
protected Map<String, String> other;
public Integer getId() {
@ -156,22 +152,6 @@ public abstract class BaseDataSourceParamDTO implements Serializable {
this.other = other;
}
public int getTestFlag() {
return testFlag;
}
public void setTestFlag(int testFlag) {
this.testFlag = testFlag;
}
public Integer getBindTestId() {
return bindTestId;
}
public void setBindTestId(Integer bindTestId) {
this.bindTestId = bindTestId;
}
/**
* Get the datasource type
* see{@link DbType}

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-starrocks/src/main/java/org/apache/dolphinscheduler/plugin/datasource/starrocks/param/StarRocksDataSourceParamDTO.java

@ -32,8 +32,6 @@ public class StarRocksDataSourceParamDTO extends BaseDataSourceParamDTO {
", port=" + port +
", database='" + database + '\'' +
", userName='" + userName + '\'' +
", testFlag=" + testFlag +
", bindTestId=" + bindTestId +
", other=" + other +
'}';
}

4
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ClickhouseDataSourceE2ETest.java

@ -65,8 +65,6 @@ public class ClickhouseDataSourceE2ETest {
private static final String jdbcParams = "";
private static final int testFlag = 1;
@BeforeAll
public static void setup() {
@ -80,7 +78,7 @@ public class ClickhouseDataSourceE2ETest {
void testCreateClickhouseDataSource() {
final DataSourcePage page = new DataSourcePage(browser);
page.createDataSource(dataSourceType, dataSourceName, dataSourceDescription, ip, port, userName, pgPassword, database, jdbcParams, testFlag);
page.createDataSource(dataSourceType, dataSourceName, dataSourceDescription, ip, port, userName, pgPassword, database, jdbcParams);
new WebDriverWait(page.driver(), 10).until(ExpectedConditions.invisibilityOfElementLocated(
new By.ByClassName("dialog-create-data-source")));

3
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/HiveDataSourceE2ETest.java

@ -64,7 +64,6 @@ public class HiveDataSourceE2ETest {
private static final String jdbcParams = "";
private static final int testFlag = 1;
@BeforeAll
public static void setup() {
@ -78,7 +77,7 @@ public class HiveDataSourceE2ETest {
void testCreateHiveDataSource() {
final DataSourcePage page = new DataSourcePage(browser);
page.createDataSource(dataSourceType, dataSourceName, dataSourceDescription, ip, port, userName, hivePassword, database, jdbcParams, testFlag);
page.createDataSource(dataSourceType, dataSourceName, dataSourceDescription, ip, port, userName, hivePassword, database, jdbcParams);
new WebDriverWait(page.driver(), 10).until(ExpectedConditions.invisibilityOfElementLocated(
new By.ByClassName("dialog-create-data-source")));

3
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/MysqlDataSourceE2ETest.java

@ -66,7 +66,6 @@ public class MysqlDataSourceE2ETest {
private static final String jdbcParams = "{\"useSSL\": false}";
private static final int testFlag = 1;
@BeforeAll
public static void setup() {
@ -80,7 +79,7 @@ public class MysqlDataSourceE2ETest {
void testCreateMysqlDataSource() {
final DataSourcePage page = new DataSourcePage(browser);
page.createDataSource(dataSourceType, dataSourceName, dataSourceDescription, ip, port, userName, mysqlPassword, database, jdbcParams, testFlag);
page.createDataSource(dataSourceType, dataSourceName, dataSourceDescription, ip, port, userName, mysqlPassword, database, jdbcParams);
new WebDriverWait(page.driver(), 10).until(ExpectedConditions.invisibilityOfElementLocated(
new By.ByClassName("dialog-create-data-source")));

3
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/PostgresDataSourceE2ETest.java

@ -65,7 +65,6 @@ public class PostgresDataSourceE2ETest {
private static final String jdbcParams = "";
private static final int testFlag = 1;
@BeforeAll
public static void setup() {
@ -79,7 +78,7 @@ public class PostgresDataSourceE2ETest {
void testCreatePostgresDataSource() {
final DataSourcePage page = new DataSourcePage(browser);
page.createDataSource(dataSourceType, dataSourceName, dataSourceDescription, ip, port, userName, pgPassword, database, jdbcParams, testFlag);
page.createDataSource(dataSourceType, dataSourceName, dataSourceDescription, ip, port, userName, pgPassword, database, jdbcParams);
new WebDriverWait(page.driver(), 10).until(ExpectedConditions.invisibilityOfElementLocated(
new By.ByClassName("dialog-create-data-source")));

3
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/SqlServerDataSourceE2ETest.java

@ -65,7 +65,6 @@ public class SqlServerDataSourceE2ETest {
private static final String jdbcParams = "";
private static final int testFlag = 1;
@BeforeAll
public static void setup() {
@ -79,7 +78,7 @@ public class SqlServerDataSourceE2ETest {
void testCreateSqlServerDataSource() {
final DataSourcePage page = new DataSourcePage(browser);
page.createDataSource(dataSourceType, dataSourceName, dataSourceDescription, ip, port, userName, pgPassword, database, jdbcParams, testFlag);
page.createDataSource(dataSourceType, dataSourceName, dataSourceDescription, ip, port, userName, pgPassword, database, jdbcParams);
new WebDriverWait(page.driver(), 10).until(ExpectedConditions.invisibilityOfElementLocated(
new By.ByClassName("dialog-create-data-source")));

16
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/datasource/DataSourcePage.java

@ -70,7 +70,7 @@ public class DataSourcePage extends NavBarPage implements NavBarPage.NavBarItem
}
public DataSourcePage createDataSource(String dataSourceType, String dataSourceName, String dataSourceDescription, String ip, String port, String userName, String password, String database,
String jdbcParams, int testFlag) {
String jdbcParams) {
buttonCreateDataSource().click();
new WebDriverWait(driver, 10).until(ExpectedConditions.visibilityOfElementLocated(
@ -89,8 +89,6 @@ public class DataSourcePage extends NavBarPage implements NavBarPage.NavBarItem
createDataSourceForm().inputUserName().sendKeys(userName);
createDataSourceForm().inputPassword().sendKeys(password);
createDataSourceForm().inputDataBase().sendKeys(database);
createDataSourceForm().radioTestDatasource().click();
if (!"".equals(jdbcParams)) {
createDataSourceForm().inputJdbcParams().sendKeys(jdbcParams);
@ -179,15 +177,6 @@ public class DataSourcePage extends NavBarPage implements NavBarPage.NavBarItem
})
private WebElement inputJdbcParams;
@FindBy(className = "radio-test-datasource")
private WebElement radioTestDatasource;
@FindBy(className = "radio-online-datasource")
private WebElement radioOnlineDatasource;
@FindBy(className = "select-bind-test-data-source-type-drop-down")
private WebElement selectBindTestDataSourceId;
@FindBy(className = "btn-submit")
private WebElement buttonSubmit;
@ -195,7 +184,6 @@ public class DataSourcePage extends NavBarPage implements NavBarPage.NavBarItem
private WebElement buttonCancel;
@FindBy(className = "btn-test-connection")
private WebElement radioTestConnection;
private WebElement btnTestConnection;
}
}

2
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java

@ -231,7 +231,5 @@ public interface ProcessService {
void forceProcessInstanceSuccessByTaskInstanceId(Integer taskInstanceId);
Integer queryTestDataSourceId(Integer onlineDataSourceId);
void saveCommandTrigger(Integer commandId, Integer processInstanceId);
}

8
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java

@ -2606,14 +2606,6 @@ public class ProcessServiceImpl implements ProcessService {
}
}
@Override
public Integer queryTestDataSourceId(Integer onlineDataSourceId) {
Integer testDataSourceId = dataSourceMapper.queryTestDataSourceId(onlineDataSourceId);
if (testDataSourceId != null)
return testDataSourceId;
return null;
}
@Override
public void saveCommandTrigger(Integer commandId, Integer processInstanceId) {
triggerRelationService.saveCommandTrigger(commandId, processInstanceId);

15
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java

@ -772,21 +772,6 @@ public class ProcessServiceTest {
}
@Test
public void testQueryTestDataSourceId() {
Integer onlineDataSourceId = 1;
// unbound testDataSourceId
Mockito.when(dataSourceMapper.queryTestDataSourceId(any(Integer.class))).thenReturn(null);
Integer result = processService.queryTestDataSourceId(onlineDataSourceId);
Assertions.assertNull(result);
// bound testDataSourceId
Integer testDataSourceId = 2;
Mockito.when(dataSourceMapper.queryTestDataSourceId(any(Integer.class))).thenReturn(testDataSourceId);
result = processService.queryTestDataSourceId(onlineDataSourceId);
Assertions.assertNotNull(result);
}
private TaskGroupQueue getTaskGroupQueue() {
TaskGroupQueue taskGroupQueue = new TaskGroupQueue();
taskGroupQueue.setTaskName("task name");

8
dolphinscheduler-ui/src/locales/en_US/datasource.ts

@ -30,21 +30,15 @@ export default {
datasource_parameter: 'Datasource Parameter',
description: 'Description',
description_tips: 'Please enter description',
test_datasource: 'Test data source',
online_datasource: 'Online data source',
bind_test_datasource: 'Bind test data source',
create_time: 'Create Time',
update_time: 'Update Time',
operation: 'Operation',
datasource_definition: 'Datasource Definition',
click_to_view: 'Click to view',
delete: 'Delete',
confirm: 'Confirm',
delete_confirm: 'Delete?',
cancel: 'Cancel',
create: 'Create',
on_line: 'Online',
test: 'Test',
edit: 'Edit',
success: 'Success',
test_connect: 'Test Connect',
@ -79,8 +73,6 @@ export default {
validation: 'Validation',
mode_tips: 'Please select a mode',
jdbc_format_tips: 'jdbc connection parameters is not a correct JSON format',
datasource_test_flag_tips: 'Please select a data source definition',
datasource_bind_test_id_tips: 'Please bind the test data source',
database_username: 'Database Username',
database_password: 'Database Password',
Azure_AD_username: 'Azure AD username',

8
dolphinscheduler-ui/src/locales/zh_CN/datasource.ts

@ -30,16 +30,10 @@ export default {
datasource_parameter: '参数',
description: '描述',
description_tips: '请输入描述',
test_datasource: '测试数据源',
online_datasource: '上线数据源',
bind_test_datasource: '绑定测试数据源',
create_time: '创建时间',
update_time: '更新时间',
operation: '操作',
click_to_view: '点击查看',
datasource_definition: '数据源定义',
on_line: '线上',
test: '测试',
delete: '删除',
confirm: '确定',
delete_confirm: '删除?',
@ -76,8 +70,6 @@ export default {
validation: '验证',
mode_tips: '请选择验证模式',
jdbc_format_tips: 'jdbc连接参数不是一个正确的JSON格式',
datasource_test_flag_tips: '请选择数据源定义',
datasource_bind_test_id_tips: '请绑定测试数据源',
database_username: '数据库用户名',
database_password: '数据库密码',
Azure_AD_username: 'Azure AD用户名',

2
dolphinscheduler-ui/src/service/modules/data-source/types.ts

@ -75,8 +75,6 @@ interface IDataSource {
database?: string
connectType?: string
other?: object
testFlag?: number
bindTestId?: number
endpoint?: string
MSIClientId?: string
dbUser?: string

44
dolphinscheduler-ui/src/utils/environmental-distinction.ts

@ -1,44 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { h } from 'vue'
import { NTag } from 'naive-ui'
export function renderEnvironmentalDistinctionCell(
testFlag: number | undefined,
t: Function
) {
if (testFlag === 0) {
return h(
NTag,
{ type: 'success', size: 'small' },
{
default: () => t('datasource.on_line')
}
)
} else if (testFlag === 1) {
return h(
NTag,
{ type: 'warning', size: 'small' },
{
default: () => t('datasource.test')
}
)
} else {
return '-'
}
}

39
dolphinscheduler-ui/src/views/datasource/list/detail.tsx

@ -65,9 +65,7 @@ const DetailModal = defineComponent({
state,
changeType,
changePort,
changeTestFlag,
resetFieldsValue,
getSameTypeTestDataSource,
setFieldsValue,
getFieldsValue
} = useForm(props.id)
@ -96,7 +94,6 @@ const DetailModal = defineComponent({
const onChangeType = changeType
const onChangePort = changePort
const onChangeTestFlag = changeTestFlag
const trim = getCurrentInstance()?.appContext.config.globalProperties.trim
@ -117,9 +114,6 @@ const DetailModal = defineComponent({
datasourceType[state.detailForm.type]
))
props.show && props.id && setFieldsValue(await queryById(props.id))
props.show &&
state.detailForm.testFlag == 0 &&
(await getSameTypeTestDataSource())
}
)
@ -142,7 +136,6 @@ const DetailModal = defineComponent({
...toRefs(state),
...toRefs(status),
onChangeType,
onChangeTestFlag,
onChangePort,
onSubmit,
onTest,
@ -174,7 +167,6 @@ const DetailModal = defineComponent({
loading,
saving,
testing,
onChangeTestFlag,
onChangePort,
onCancel,
onTest,
@ -651,37 +643,6 @@ const DetailModal = defineComponent({
)}`}
/>
</NFormItem>
<NFormItem
label={t('datasource.datasource_definition')}
path='testFlag'
show-require-mark
>
<NRadioGroup
v-model={[detailForm.testFlag, 'value']}
onUpdate:value={onChangeTestFlag}
>
<NSpace>
<NRadio value={1} class='radio-test-datasource'>
{t('datasource.test_datasource')}
</NRadio>
<NRadio value={0} class='radio-online-datasource'>
{t('datasource.online_datasource')}
</NRadio>
</NSpace>
</NRadioGroup>
</NFormItem>
<NFormItem
v-show={detailForm.testFlag == 0}
label={t('datasource.bind_test_datasource')}
path='bindTestId'
show-require-mark
>
<NSelect
class='select-bind-test-data-source-type-drop-down'
v-model={[detailForm.bindTestId, 'value']}
options={this.bindTestDataSourceExample}
/>
</NFormItem>
<NFormItem
v-show={showPublicKey}
label='PublicKey'

10
dolphinscheduler-ui/src/views/datasource/list/use-columns.ts

@ -28,13 +28,12 @@ import {
import { EditOutlined, DeleteOutlined } from '@vicons/antd'
import JsonHighlight from './json-highlight'
import ButtonLink from '@/components/button-link'
import type { IDataSource, TableColumns } from './types'
import {
COLUMN_WIDTH_CONFIG,
calculateTableWidth,
DefaultTableWidth
} from '@/common/column-width-config'
import { renderEnvironmentalDistinctionCell } from '@/utils/environmental-distinction'
import type { TableColumns } from './types'
export function useColumns(onCallback: Function) {
const { t } = useI18n()
@ -62,13 +61,6 @@ export function useColumns(onCallback: Function) {
key: 'type',
width: 180
},
{
title: t('datasource.datasource_definition'),
key: 'testFlag',
width: 140,
render: (_row: IDataSource) =>
renderEnvironmentalDistinctionCell(_row.testFlag, t)
},
{
title: t('datasource.datasource_parameter'),
key: 'parameter',

57
dolphinscheduler-ui/src/views/datasource/list/use-form.ts

@ -17,10 +17,7 @@
import { reactive, ref } from 'vue'
import { useI18n } from 'vue-i18n'
import {
getKerberosStartupState,
queryDataSourceList
} from '@/service/modules/data-source'
import { getKerberosStartupState } from '@/service/modules/data-source'
import type { FormRules } from 'naive-ui'
import type {
IDataSourceDetail,
@ -30,7 +27,7 @@ import type {
IDataSource
} from './types'
import utils from '@/utils'
import type { TypeReq } from '@/service/modules/data-source/types'
export function useForm(id?: number) {
const { t } = useI18n()
@ -51,8 +48,6 @@ export function useForm(id?: number) {
database: '',
connectType: '',
other: '',
testFlag: -1,
bindTestId: undefined,
endpoint: '',
MSIClientId: '',
dbUser: '',
@ -73,7 +68,6 @@ export function useForm(id?: number) {
showDataBaseName: true,
showJDBCConnectParameters: true,
showPublicKey: false,
bindTestDataSourceExample: [] as { label: string; value: number }[],
rules: {
name: {
trigger: ['input'],
@ -169,22 +163,6 @@ export function useForm(id?: number) {
}
}
},
testFlag: {
trigger: ['input'],
validator() {
if (-1 === state.detailForm.testFlag) {
return new Error(t('datasource.datasource_test_flag_tips'))
}
}
},
bindTestId: {
trigger: ['input'],
validator() {
if (0 === state.detailForm.testFlag && !state.detailForm.bindTestId) {
return new Error(t('datasource.datasource_bind_test_id_tips'))
}
}
},
endpoint: {
trigger: ['input'],
validator() {
@ -286,10 +264,6 @@ export function useForm(id?: number) {
state.showJDBCConnectParameters = true
state.showPublicKey = false
}
if (state.detailForm.id === undefined) {
await getSameTypeTestDataSource()
}
}
const changePort = async () => {
@ -297,31 +271,6 @@ export function useForm(id?: number) {
const currentDataBaseOption = datasourceType[state.detailForm.type]
currentDataBaseOption.previousPort = state.detailForm.port
}
const changeTestFlag = async (testFlag: IDataBase) => {
if (testFlag) {
state.detailForm.bindTestId = undefined
}
// @ts-ignore
if (state.detailForm.id !== undefined && testFlag === 0) {
await getSameTypeTestDataSource()
}
}
const getSameTypeTestDataSource = async () => {
const params = { type: state.detailForm.type, testFlag: 1 } as TypeReq
const result = await queryDataSourceList(params)
state.bindTestDataSourceExample = result
.filter((value: { label: string; value: string }) => {
// @ts-ignore
if (state.detailForm.id && state.detailForm.id === value.id)
return false
return true
})
.map((TestDataSourceExample: { name: string; id: number }) => ({
label: TestDataSourceExample.name,
value: TestDataSourceExample.id
}))
}
const resetFieldsValue = () => {
state.detailForm = { ...initialValues }
@ -341,9 +290,7 @@ export function useForm(id?: number) {
state,
changeType,
changePort,
changeTestFlag,
resetFieldsValue,
getSameTypeTestDataSource,
setFieldsValue,
getFieldsValue
}

8
dolphinscheduler-ui/src/views/projects/task/instance/use-table.ts

@ -39,7 +39,6 @@ import {
DefaultTableWidth
} from '@/common/column-width-config'
import type { Router, TaskInstancesRes, IRecord, ITaskState } from './types'
import { renderEnvironmentalDistinctionCell } from '@/utils/environmental-distinction'
export function useTable() {
const { t } = useI18n()
@ -128,13 +127,6 @@ export function useTable() {
key: 'executorName',
...COLUMN_WIDTH_CONFIG['name']
},
{
title: t('project.task.operating_environment'),
key: 'testFlag',
width: 160,
render: (_row: IRecord) =>
renderEnvironmentalDistinctionCell(_row.testFlag, t)
},
{
title: t('project.task.node_type'),
key: 'taskType',

9
dolphinscheduler-ui/src/views/projects/workflow/instance/use-table.ts

@ -43,7 +43,6 @@ import type { Router } from 'vue-router'
import type { IWorkflowInstance } from '@/service/modules/process-instances/types'
import type { ICountDownParam } from './types'
import type { ExecuteReq } from '@/service/modules/executors/types'
import { renderEnvironmentalDistinctionCell } from '@/utils/environmental-distinction'
import { IWorkflowExecutionState } from '@/common/types'
export function useTable() {
@ -126,14 +125,6 @@ export function useTable() {
render: (_row: IWorkflowInstance) =>
renderWorkflowStateCell(_row.state, t)
},
{
title: t('project.workflow.operating_environment'),
key: 'testFlag',
width: 160,
className: 'workflow-testFlag',
render: (_row: IWorkflowInstance) =>
renderEnvironmentalDistinctionCell(_row.testFlag, t)
},
{
title: t('project.workflow.run_type'),
key: 'commandType',

Loading…
Cancel
Save