From 00ff88ef1c4a8af8d2dc1324543338629dd3ca6b Mon Sep 17 00:00:00 2001 From: Simon <3656562@qq.com> Date: Thu, 19 Mar 2020 18:56:58 +0800 Subject: [PATCH 01/22] support custom datax configuration (#2224) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix #1441 * support custom datax config * support datax custom config * support datax custom config * support datax custom config Co-authored-by: 张世鸣 --- .../api/utils/CheckUtilsTest.java | 1 + .../common/task/datax/DataxParameters.java | 46 ++++++++++++++--- .../server/worker/task/datax/DataxTask.java | 39 ++++++++++++--- .../worker/task/datax/DataxTaskTest.java | 49 ++++++++++++++----- 4 files changed, 106 insertions(+), 29 deletions(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java index 24a0ed31d6..308ed8e9b6 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java @@ -211,6 +211,7 @@ public class CheckUtilsTest { // DataxParameters DataxParameters dataxParameters = new DataxParameters(); assertFalse(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(dataxParameters), TaskType.DATAX.toString())); + dataxParameters.setCustomConfig(0); dataxParameters.setDataSource(111); dataxParameters.setDataTarget(333); dataxParameters.setSql("sql"); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java index 95dd505c02..f153360d63 100755 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java @@ -27,6 +27,16 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters; */ public class DataxParameters extends AbstractParameters { + /** + * if custom json config，eg 0, 1 + */ + private Integer customConfig; + + /** + * if customConfig eq 1 ,then json is usable + */ + private String json; + /** * data source type，eg MYSQL, POSTGRES ... */ @@ -77,6 +87,22 @@ public class DataxParameters extends AbstractParameters { */ private int jobSpeedRecord; + public Integer getCustomConfig() { + return customConfig; + } + + public void setCustomConfig(Integer customConfig) { + this.customConfig = customConfig; + } + + public String getJson() { + return json; + } + + public void setJson(String json) { + this.json = json; + } + public String getDsType() { return dsType; } @@ -157,16 +183,18 @@ public class DataxParameters extends AbstractParameters { this.jobSpeedRecord = jobSpeedRecord; } + @Override public boolean checkParameters() { - if (!(dataSource != 0 - && dataTarget != 0 - && StringUtils.isNotEmpty(sql) - && StringUtils.isNotEmpty(targetTable))) { - return false; + if (customConfig == null) return false; + if (customConfig == 0) { + return dataSource != 0 + && dataTarget != 0 + && StringUtils.isNotEmpty(sql) + && StringUtils.isNotEmpty(targetTable); + } else { + return StringUtils.isNotEmpty(json); } - - return true; } @Override @@ -177,7 +205,9 @@ public class DataxParameters extends AbstractParameters { @Override public String toString() { return "DataxParameters{" + - "dsType='" + dsType + '\'' + + "customConfig=" + customConfig + + ", json='" + json + '\'' + + ", dsType='" + dsType + '\'' + ", dataSource=" + dataSource + ", dtType='" + dtType + '\'' + ", dataTarget=" + dataTarget + diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index 952030ea62..8083bb6869 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -192,24 +192,47 @@ public class DataxTask extends AbstractTask { throws Exception { // generate json String fileName = String.format("%s/%s_job.json", taskDir, taskProps.getTaskAppId()); + String json; Path path = new File(fileName).toPath(); if (Files.exists(path)) { return fileName; } - JSONObject job = new JSONObject(); - job.put("content", buildDataxJobContentJson()); - job.put("setting", buildDataxJobSettingJson()); - JSONObject root = new JSONObject(); - root.put("job", job); - root.put("core", buildDataxCoreJson()); - logger.debug("datax job json : {}", root.toString()); + if (dataXParameters.getCustomConfig() == 1){ + + json = dataXParameters.getJson().replaceAll("\\r\\n", "\n"); + + /** + * combining local and global parameters + */ + Map paramsMap = ParamUtils.convert(taskProps.getUserDefParamsMap(), + taskProps.getDefinedParams(), + dataXParameters.getLocalParametersMap(), + taskProps.getCmdTypeIfComplement(), + taskProps.getScheduleTime()); + if (paramsMap != null){ + json = ParameterUtils.convertParameterPlaceholders(json, ParamUtils.convert(paramsMap)); + } + + }else { + + JSONObject job = new JSONObject(); + job.put("content", buildDataxJobContentJson()); + job.put("setting", buildDataxJobSettingJson()); + + JSONObject root = new JSONObject(); + root.put("job", job); + root.put("core", buildDataxCoreJson()); + json = root.toString(); + } + + logger.debug("datax job json : {}", json); // create datax json file - FileUtils.writeStringToFile(new File(fileName), root.toString(), StandardCharsets.UTF_8); + FileUtils.writeStringToFile(new File(fileName), json, StandardCharsets.UTF_8); return fileName; } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java index bd7f27530a..c2dbd268e6 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java @@ -44,6 +44,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; +import static org.apache.dolphinscheduler.common.enums.CommandType.START_PROCESS; + /** * DataxTask Tester. */ @@ -59,6 +61,8 @@ public class DataxTaskTest { private ApplicationContext applicationContext; + private TaskProps props = new TaskProps(); + @Before public void before() throws Exception { @@ -70,7 +74,6 @@ public class DataxTaskTest { springApplicationContext.setApplicationContext(applicationContext); Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService); - TaskProps props = new TaskProps(); props.setTaskDir("/tmp"); props.setTaskAppId(String.valueOf(System.currentTimeMillis())); props.setTaskInstId(1); @@ -78,10 +81,8 @@ public class DataxTaskTest { props.setEnvFile(".dolphinscheduler_env.sh"); props.setTaskStartTime(new Date()); props.setTaskTimeout(0); - props.setTaskParams( - "{\"targetTable\":\"test\",\"postStatements\":[],\"jobSpeedByte\":1024,\"jobSpeedRecord\":1000,\"dtType\":\"MYSQL\",\"datasource\":1,\"dsType\":\"MYSQL\",\"datatarget\":2,\"jobSpeedByte\":0,\"sql\":\"select 1 as test from dual\",\"preStatements\":[\"delete from test\"],\"postStatements\":[\"delete from test\"]}"); - dataxTask = PowerMockito.spy(new DataxTask(props, logger)); - dataxTask.init(); + props.setCmdTypeIfComplement(START_PROCESS); + setTaskParems(0); Mockito.when(processService.findDataSourceById(1)).thenReturn(getDataSource()); Mockito.when(processService.findDataSourceById(2)).thenReturn(getDataSource()); @@ -91,6 +92,22 @@ public class DataxTaskTest { Mockito.when(shellCommandExecutor.run(fileName, processService)).thenReturn(0); } + private void setTaskParems(Integer customConfig) { + if (customConfig == 1) { + props.setTaskParams( + "{\"customConfig\":1, \"localParams\":[{\"prop\":\"test\",\"value\":\"38294729\"}],\"json\":\"{\\\"job\\\":{\\\"setting\\\":{\\\"speed\\\":{\\\"byte\\\":1048576},\\\"errorLimit\\\":{\\\"record\\\":0,\\\"percentage\\\":0.02}},\\\"content\\\":[{\\\"reader\\\":{\\\"name\\\":\\\"rdbmsreader\\\",\\\"parameter\\\":{\\\"username\\\":\\\"xxx\\\",\\\"password\\\":\\\"${test}\\\",\\\"column\\\":[\\\"id\\\",\\\"name\\\"],\\\"splitPk\\\":\\\"pk\\\",\\\"connection\\\":[{\\\"querySql\\\":[\\\"SELECT * from dual\\\"],\\\"jdbcUrl\\\":[\\\"jdbc:dm://ip:port/database\\\"]}],\\\"fetchSize\\\":1024,\\\"where\\\":\\\"1 = 1\\\"}},\\\"writer\\\":{\\\"name\\\":\\\"streamwriter\\\",\\\"parameter\\\":{\\\"print\\\":true}}}]}}\"}"); + +// "{\"customConfig\":1,\"json\":\"{\\\"job\\\":{\\\"setting\\\":{\\\"speed\\\":{\\\"byte\\\":1048576},\\\"errorLimit\\\":{\\\"record\\\":0,\\\"percentage\\\":0.02}},\\\"content\\\":[{\\\"reader\\\":{\\\"name\\\":\\\"rdbmsreader\\\",\\\"parameter\\\":{\\\"username\\\":\\\"xxx\\\",\\\"password\\\":\\\"xxx\\\",\\\"column\\\":[\\\"id\\\",\\\"name\\\"],\\\"splitPk\\\":\\\"pk\\\",\\\"connection\\\":[{\\\"querySql\\\":[\\\"SELECT * from dual\\\"],\\\"jdbcUrl\\\":[\\\"jdbc:dm://ip:port/database\\\"]}],\\\"fetchSize\\\":1024,\\\"where\\\":\\\"1 = 1\\\"}},\\\"writer\\\":{\\\"name\\\":\\\"streamwriter\\\",\\\"parameter\\\":{\\\"print\\\":true}}}]}}\"}"); + } else { + props.setTaskParams( + "{\"customConfig\":0,\"targetTable\":\"test\",\"postStatements\":[],\"jobSpeedByte\":1024,\"jobSpeedRecord\":1000,\"dtType\":\"MYSQL\",\"datasource\":1,\"dsType\":\"MYSQL\",\"datatarget\":2,\"jobSpeedByte\":0,\"sql\":\"select 1 as test from dual\",\"preStatements\":[\"delete from test\"],\"postStatements\":[\"delete from test\"]}"); + + } + + dataxTask = PowerMockito.spy(new DataxTask(props, logger)); + dataxTask.init(); + } + private DataSource getDataSource() { DataSource dataSource = new DataSource(); dataSource.setType(DbType.MYSQL); @@ -102,7 +119,7 @@ public class DataxTaskTest { private ProcessInstance getProcessInstance() { ProcessInstance processInstance = new ProcessInstance(); - processInstance.setCommandType(CommandType.START_PROCESS); + processInstance.setCommandType(START_PROCESS); processInstance.setScheduleTime(new Date()); return processInstance; } @@ -229,18 +246,24 @@ public class DataxTaskTest { */ @Test public void testBuildDataxJsonFile() - throws Exception { + throws Exception { try { - Method method = DataxTask.class.getDeclaredMethod("buildDataxJsonFile"); - method.setAccessible(true); - String filePath = (String) method.invoke(dataxTask, null); - Assert.assertNotNull(filePath); - } - catch (Exception e) { + setTaskParems(1); + buildDataJson(); + setTaskParems(0); + buildDataJson(); + } catch (Exception e) { Assert.fail(e.getMessage()); } } + public void buildDataJson() throws Exception { + Method method = DataxTask.class.getDeclaredMethod("buildDataxJsonFile"); + method.setAccessible(true); + String filePath = (String) method.invoke(dataxTask, null); + Assert.assertNotNull(filePath); + } + /** * Method: buildDataxJobContentJson() */ From 07149714535649f6d2c768a38458af94f9e90f4f Mon Sep 17 00:00:00 2001 From: John Liu <517356906@qq.com> Date: Thu, 19 Mar 2020 19:26:03 +0800 Subject: [PATCH 02/22] Optimizing code (#2240) * updated * optimized the code * Delete ParameterUtils.java * rollback source code * rollback source code * rollback source code Co-authored-by: dailidong --- CONTRIBUTING.md | 94 +++++++++++++------ .../apache/dolphinscheduler/dao/AlertDao.java | 27 ++---- 2 files changed, 74 insertions(+), 47 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8ed9aac897..e02ed113c4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,35 +1,53 @@ -* First from the remote repository *https://github.com/apache/incubator-dolphinscheduler.git* fork code to your own repository -* there are three branches in the remote repository currently: - * master normal delivery branch - After the stable version is released, the code for the stable version branch is merged into the master branch. +# Development - * dev daily development branch - The daily development branch, the newly submitted code can pull requests to this branch. +Start by forking the dolphinscheduler GitHub repository, make changes in a branch and then send a pull request. +## Set up your dolphinscheduler GitHub Repository -* Clone your own warehouse to your local +There are three branches in the remote repository currently: + - `master` : normal delivery branch. After the stable version is released, the code for the stable version branch is merged into the master branch. + + - `dev` : daily development branch. The daily development branch, the newly submitted code can pull requests to this branch. + + - `x.x.x-release` : the stable release version. - `git clone https://github.com/apache/incubator-dolphinscheduler.git` +So, you should fork the `dev` branch. -* Add remote repository address, named upstream +After forking the [dolphinscheduler upstream source repository](https://github.com/apache/incubator-dolphinscheduler/fork) to your personal repository, you can set your personal development environment. - `git remote add upstream https://github.com/apache/incubator-dolphinscheduler.git` +```sh +$ cd +$ git clone < your personal forked dolphinscheduler repo> +$ cd incubator-dolphinscheduler +``` -* View repository: +## Set git remote as ``upstream`` - `git remote -v` +Add remote repository address, named upstream -> There will be two repositories at this time: origin (your own warehouse) and upstream (remote repository) +```sh +git remote add upstream https://github.com/apache/incubator-dolphinscheduler.git +``` -* Get/update remote repository code (already the latest code, skip it) +View repository: - `git fetch upstream` +```sh +git remote -v +``` +There will be two repositories at this time: origin (your own warehouse) and upstream (remote repository) -* Synchronize remote repository code to local repository +Get/update remote repository code (already the latest code, skip it). + +```sh +git fetch upstream ``` + +Synchronize remote repository code to local repository + +```sh git checkout origin/dev git merge --no-ff upstream/dev ``` @@ -41,24 +59,46 @@ git checkout -b dev-1.0 upstream/dev-1.0 git push --set-upstream origin dev1.0 ``` -* After modifying the code locally, submit it to your own repository: +## Create your feature branch +Before making code changes, make sure you create a separate branch for them. + +```sh +$ git checkout -b +``` + +## Commit changes +After modifying the code locally, submit it to your own repository: + +```sh + +git commit -m 'information about your feature' +``` + +## Push to the branch + + +Push your locally committed changes to the remote origin (your fork). -`git commit -m 'test commit'` -`git push` +``` +$ git push origin +``` + +## Create a pull request -* Submit changes to the remote repository +After submitting changes to your remote repository, you should click on the new pull request On the following github page. -* On the github page, click on the new pull request.

- - + +

+ + +Select the modified local branch and the branch to merge past to create a pull request. -* Select the modified local branch and the branch to merge past to create a pull request.

- - + +

-* Next, the administrator is responsible for **merging** to complete the pull request +Next, the administrator is responsible for **merging** to complete the pull request. diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java index 2ba93d7607..8546f01aad 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java @@ -99,13 +99,7 @@ public class AlertDao extends AbstractBaseDao { String content = String.format("[{'type':'%s','host':'%s','event':'server down','warning level':'serious'}]", serverType, host); alert.setTitle("Fault tolerance warning"); - alert.setShowType(ShowType.TABLE); - alert.setContent(content); - alert.setAlertType(AlertType.EMAIL); - alert.setAlertGroupId(alertgroupId); - alert.setCreateTime(new Date()); - alert.setUpdateTime(new Date()); - alertMapper.insert(alert); + saveTaskTimeoutAlert(alert, content, alertgroupId, null, null); } /** @@ -121,6 +115,11 @@ public class AlertDao extends AbstractBaseDao { String content = String.format("[{'id':'%d','name':'%s','event':'timeout','warnLevel':'middle'}]", processInstance.getId(), processInstance.getName()); alert.setTitle("Process Timeout Warn"); + saveTaskTimeoutAlert(alert, content, alertgroupId, receivers, receiversCc); + } + + private void saveTaskTimeoutAlert(Alert alert, String content, int alertgroupId, + String receivers, String receiversCc){ alert.setShowType(ShowType.TABLE); alert.setContent(content); alert.setAlertType(AlertType.EMAIL); @@ -150,19 +149,7 @@ public class AlertDao extends AbstractBaseDao { String content = String.format("[{'process instance id':'%d','task name':'%s','task id':'%d','task name':'%s'," + "'event':'timeout','warnLevel':'middle'}]", processInstanceId, processInstanceName, taskId, taskName); alert.setTitle("Task Timeout Warn"); - alert.setShowType(ShowType.TABLE); - alert.setContent(content); - alert.setAlertType(AlertType.EMAIL); - alert.setAlertGroupId(alertgroupId); - if (StringUtils.isNotEmpty(receivers)) { - alert.setReceivers(receivers); - } - if (StringUtils.isNotEmpty(receiversCc)) { - alert.setReceiversCc(receiversCc); - } - alert.setCreateTime(new Date()); - alert.setUpdateTime(new Date()); - alertMapper.insert(alert); + saveTaskTimeoutAlert(alert, content, alertgroupId, receivers, receiversCc); } /** From f4897b563d16536647f5dca5c5c3039b8d8e52ca Mon Sep 17 00:00:00 2001 From: tswstarplanet Date: Thu, 19 Mar 2020 23:56:26 +0800 Subject: [PATCH 03/22] fix blocker problem (#2244) * fix blocker problem * use error level to log exception --- .../org/apache/dolphinscheduler/remote/NettyRemotingClient.java | 2 +- .../dolphinscheduler/remote/handler/NettyServerHandler.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java index 96258d752a..d3def39c02 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java @@ -301,7 +301,7 @@ public class NettyRemotingClient { return channel; } } catch (Exception ex) { - logger.info("connect to {} error {}", address, ex); + logger.error("connect to {} error", address, ex); } return null; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java index 88cd2d5518..c601aa9891 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java @@ -117,7 +117,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { try { pair.getLeft().process(channel, msg); } catch (Throwable ex) { - logger.error("process msg {} error : {}", msg, ex); + logger.error("process msg {} error", msg, ex); } } }; From 74bd33301ff13ba4339a840c9f1bbbc95845c777 Mon Sep 17 00:00:00 2001 From: jin Date: Fri, 20 Mar 2020 17:43:44 +0800 Subject: [PATCH 04/22] feat:optimizing code (#2245) * feat:optimizing code * add licence and format code --- .../api/controller/AlertGroupController.java | 24 ++++++--- .../api/service/AlertGroupService.java | 43 ++++++--------- .../api/service/UserAlertGroupService.java | 38 +++++++++++++ .../api/service/AlertGroupServiceTest.java | 47 +++++++++------- .../service/UserAlertGroupServiceTest.java | 53 +++++++++++++++++++ pom.xml | 1 + 6 files changed, 151 insertions(+), 55 deletions(-) create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UserAlertGroupService.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UserAlertGroupServiceTest.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java index e9bffa510b..140434ee43 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java @@ -93,11 +93,11 @@ public class AlertGroupController extends BaseController{ public Result list(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser) { logger.info("login user {}, query all alertGroup", loginUser.getUserName()); - try{ + try { HashMap result = alertGroupService.queryAlertgroup(); return returnDataList(result); - }catch (Exception e){ - logger.error(Status.QUERY_ALL_ALERTGROUP_ERROR.getMsg(),e); + } catch (Exception e) { + logger.error(Status.QUERY_ALL_ALERTGROUP_ERROR.getMsg(), e); return error(Status.QUERY_ALL_ALERTGROUP_ERROR.getCode(), Status.QUERY_ALL_ALERTGROUP_ERROR.getMsg()); } } @@ -214,12 +214,20 @@ public class AlertGroupController extends BaseController{ @GetMapping(value = "/verify-group-name") @ResponseStatus(HttpStatus.OK) public Result verifyGroupName(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value ="groupName") String groupName - ) { - logger.info("login user {}, verfiy group name: {}", - loginUser.getUserName(),groupName); + @RequestParam(value ="groupName") String groupName) { + logger.info("login user {}, verify group name: {}", loginUser.getUserName(), groupName); - return alertGroupService.verifyGroupName(loginUser, groupName); + boolean exist= alertGroupService.existGroupName(groupName); + Result result = new Result(); + if (exist) { + logger.error("group {} has exist, can't create again.", groupName); + result.setCode(Status.ALERT_GROUP_EXIST.getCode()); + result.setMsg(Status.ALERT_GROUP_EXIST.getMsg()); + } else { + result.setCode(Status.SUCCESS.getCode()); + result.setMsg(Status.SUCCESS.getMsg()); + } + return result; } /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java index 70310b6331..001a10d08a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java @@ -16,17 +16,17 @@ */ package org.apache.dolphinscheduler.api.service; +import java.util.*; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; -import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.AlertType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.AlertGroup; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.UserAlertGroup; import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper; -import org.apache.dolphinscheduler.dao.mapper.UserAlertGroupMapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.slf4j.Logger; @@ -35,11 +35,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - /** * alert group service */ @@ -52,8 +47,7 @@ public class AlertGroupService extends BaseService{ private AlertGroupMapper alertGroupMapper; @Autowired - private UserAlertGroupMapper userAlertGroupMapper; - + private UserAlertGroupService userAlertGroupService; /** * query alert group list * @@ -122,7 +116,7 @@ public class AlertGroupService extends BaseService{ alertGroup.setCreateTime(now); alertGroup.setUpdateTime(now); - // insert + // insert int insert = alertGroupMapper.insert(alertGroup); if (insert > 0) { @@ -199,7 +193,7 @@ public class AlertGroupService extends BaseService{ return result; } - userAlertGroupMapper.deleteByAlertgroupId(id); + userAlertGroupService.deleteByAlertGroupId(id); alertGroupMapper.deleteById(id); putMsg(result, Status.SUCCESS); return result; @@ -223,22 +217,26 @@ public class AlertGroupService extends BaseService{ return result; } - userAlertGroupMapper.deleteByAlertgroupId(alertgroupId); + userAlertGroupService.deleteByAlertGroupId(alertgroupId); if (StringUtils.isEmpty(userIds)) { putMsg(result, Status.SUCCESS); return result; } String[] userIdsArr = userIds.split(","); - + Date now = new Date(); + List alertGroups = new ArrayList<>(userIds.length()); for (String userId : userIdsArr) { - Date now = new Date(); UserAlertGroup userAlertGroup = new UserAlertGroup(); userAlertGroup.setAlertgroupId(alertgroupId); userAlertGroup.setUserId(Integer.parseInt(userId)); userAlertGroup.setCreateTime(now); userAlertGroup.setUpdateTime(now); - userAlertGroupMapper.insert(userAlertGroup); + alertGroups.add(userAlertGroup); + } + + if (CollectionUtils.isNotEmpty(alertGroups)) { + userAlertGroupService.saveBatch(alertGroups); } putMsg(result, Status.SUCCESS); @@ -248,22 +246,11 @@ public class AlertGroupService extends BaseService{ /** * verify group name exists * - * @param loginUser login user * @param groupName group name * @return check result code */ - public Result verifyGroupName(User loginUser, String groupName) { - Result result = new Result(); + public boolean existGroupName(String groupName) { List alertGroup = alertGroupMapper.queryByGroupName(groupName); - if (alertGroup != null && alertGroup.size() > 0) { - logger.error("group {} has exist, can't create again.", groupName); - result.setCode(Status.ALERT_GROUP_EXIST.getCode()); - result.setMsg(Status.ALERT_GROUP_EXIST.getMsg()); - } else { - result.setCode(Status.SUCCESS.getCode()); - result.setMsg(Status.SUCCESS.getMsg()); - } - - return result; + return CollectionUtils.isNotEmpty(alertGroup); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UserAlertGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UserAlertGroupService.java new file mode 100644 index 0000000000..502185709f --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UserAlertGroupService.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import org.apache.dolphinscheduler.dao.entity.UserAlertGroup; +import org.apache.dolphinscheduler.dao.mapper.UserAlertGroupMapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * + */ +@Service +public class UserAlertGroupService extends ServiceImpl { + + @Autowired + private UserAlertGroupMapper userAlertGroupMapper; + + boolean deleteByAlertGroupId(Integer groupId) { + return userAlertGroupMapper.deleteByAlertgroupId(groupId) >= 1; + } + +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java index 4a31902af9..ab7dac4d60 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java @@ -18,9 +18,12 @@ package org.apache.dolphinscheduler.api.service; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; -import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.enums.UserType; @@ -31,9 +34,12 @@ import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper; import org.apache.dolphinscheduler.dao.mapper.UserAlertGroupMapper; import org.junit.After; import org.junit.Assert; +import static org.junit.Assert.assertEquals; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import static org.mockito.ArgumentMatchers.*; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; @@ -41,14 +47,6 @@ import org.mockito.junit.MockitoJUnitRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; - @RunWith(MockitoJUnitRunner.class) public class AlertGroupServiceTest { @@ -60,6 +58,8 @@ public class AlertGroupServiceTest { private AlertGroupMapper alertGroupMapper; @Mock private UserAlertGroupMapper userAlertGroupMapper; + @Mock + UserAlertGroupService userAlertGroupService; private String groupName = "AlertGroupServiceTest"; @@ -160,25 +160,34 @@ public class AlertGroupServiceTest { } + @Test - public void testGrantUser(){ + public void testGrantUser() { + + Integer groupId = 1; + + ArgumentCaptor groupArgument = ArgumentCaptor.forClass(Integer.class); + + Mockito.when(userAlertGroupService.deleteByAlertGroupId(anyInt())).thenReturn(true); + + Map result = alertGroupService.grantUser(getLoginUser(), groupId, "123,321"); + Mockito.verify(userAlertGroupService).deleteByAlertGroupId(groupArgument.capture()); - Map result = alertGroupService.grantUser(getLoginUser(),1,"123,321"); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + assertEquals(groupArgument.getValue(), groupId); + assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } + @Test - public void testVerifyGroupName(){ + public void testVerifyGroupName() { //group name not exist - Result result = alertGroupService.verifyGroupName(getLoginUser(), groupName); - logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + boolean result = alertGroupService.existGroupName(groupName); + Assert.assertFalse(result); Mockito.when(alertGroupMapper.queryByGroupName(groupName)).thenReturn(getList()); //group name exist - result = alertGroupService.verifyGroupName(getLoginUser(), groupName); - logger.info(result.toString()); - Assert.assertEquals(Status.ALERT_GROUP_EXIST.getMsg(),result.getMsg()); + result = alertGroupService.existGroupName(groupName); + Assert.assertTrue(result); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UserAlertGroupServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UserAlertGroupServiceTest.java new file mode 100644 index 0000000000..24b1d5a98b --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UserAlertGroupServiceTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import org.apache.dolphinscheduler.dao.mapper.UserAlertGroupMapper; +import static org.junit.Assert.assertEquals; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +/** + * + */ +@RunWith(MockitoJUnitRunner.class) +public class UserAlertGroupServiceTest { + + @InjectMocks + UserAlertGroupService userAlertGroupService; + + @Mock + UserAlertGroupMapper userAlertGroupMapper; + + @Test + public void deleteByAlertGroupId() { + + Integer groupId = 1; + userAlertGroupService.deleteByAlertGroupId(groupId); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Integer.class); + + Mockito.verify(userAlertGroupMapper).deleteByAlertgroupId(argumentCaptor.capture()); + assertEquals(argumentCaptor.getValue(), groupId); + + } + +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 412a28eda7..dcb712a578 100644 --- a/pom.xml +++ b/pom.xml @@ -707,6 +707,7 @@ **/api/service/TenantServiceTest.java **/api/service/WorkerGroupServiceTest.java **/api/service/AlertGroupServiceTest.java + **/api/service/UserAlertGroupServiceTest.java **/api/service/ProjectServiceTest.java **/api/service/ProcessDefinitionServiceTest.java **/api/service/UdfFuncServiceTest.java From 4318e860d5c05ecb2a34b9c29eb75afce6cba727 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Fri, 20 Mar 2020 18:46:25 +0800 Subject: [PATCH 05/22] Adapting partial code(file name start with R) to the sonar cloud rule (#2243) --- .../api/controller/ResourcesController.java | 8 +++--- .../api/service/ResourcesService.java | 25 ++++++------------- .../common/utils/PropertyUtils.java | 2 +- .../dao/mapper/ResourceMapperTest.java | 12 ++++----- .../dao/mapper/ResourceUserMapperTest.java | 4 +-- .../remote/future/ResponseFuture.java | 8 +++--- .../server/utils/RemoveZKNode.java | 2 +- 7 files changed, 25 insertions(+), 36 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java index 7bac6614ee..f28ba100f2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java @@ -144,7 +144,7 @@ public class ResourcesController extends BaseController{ @RequestParam(value ="type") ResourceType type ){ try{ - logger.info("query resource list, login user:{}, resource type:{}", loginUser.getUserName(), type.toString()); + logger.info("query resource list, login user:{}, resource type:{}", loginUser.getUserName(), type); Map result = resourceService.queryResourceList(loginUser, type); return returnDataList(result); }catch (Exception e){ @@ -180,7 +180,7 @@ public class ResourcesController extends BaseController{ ){ try{ logger.info("query resource list, login user:{}, resource type:{}, search value:{}", - loginUser.getUserName(), type.toString(), searchVal); + loginUser.getUserName(), type, searchVal); Map result = checkPageParams(pageNo, pageSize); if(result.get(Constants.STATUS) != Status.SUCCESS){ return returnDataListPaging(result); @@ -426,8 +426,6 @@ public class ResourcesController extends BaseController{ @RequestParam(value = "resourceId") int resourceId) { logger.info("login user {}, create udf function, type: {}, funcName: {},argTypes: {} ,database: {},desc: {},resourceId: {}", loginUser.getUserName(),type, funcName, argTypes,database,description, resourceId); - Result result = new Result(); - try { return udfFuncService.createUdfFunction(loginUser,funcName,className,argTypes,database,description,type,resourceId); } catch (Exception e) { @@ -563,7 +561,7 @@ public class ResourcesController extends BaseController{ public Result queryResourceList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("type") UdfType type){ try{ - logger.info("query datasource list, user:{}, type:{}", loginUser.getUserName(), type.toString()); + logger.info("query datasource list, user:{}, type:{}", loginUser.getUserName(), type); Map result = udfFuncService.queryResourceList(loginUser,type.ordinal()); return returnDataList(result); }catch (Exception e){ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index c1adb8874d..f46eda757a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -24,10 +24,7 @@ import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResourceType; -import org.apache.dolphinscheduler.common.utils.FileUtils; -import org.apache.dolphinscheduler.common.utils.HadoopUtils; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.UdfFunc; @@ -141,7 +138,7 @@ public class ResourcesService extends BaseService { putMsg(result, Status.SUCCESS); Map dataMap = new BeanMap(resource); - Map resultMap = new HashMap(); + Map resultMap = new HashMap<>(); for (Map.Entry entry: dataMap.entrySet()) { if (!"class".equalsIgnoreCase(entry.getKey().toString())) { resultMap.put(entry.getKey().toString(), entry.getValue()); @@ -171,12 +168,8 @@ public class ResourcesService extends BaseService { * @return true if resource exists */ private boolean checkResourceExists(String alias, int userId, int type ){ - List resources = resourcesMapper.queryResourceList(alias, userId, type); - if (resources != null && resources.size() > 0) { - return true; - } - return false; + return CollectionUtils.isNotEmpty(resources); } @@ -237,7 +230,7 @@ public class ResourcesService extends BaseService { //get the file suffix String originResourceName = resource.getAlias(); - String suffix = originResourceName.substring(originResourceName.lastIndexOf(".")); + String suffix = originResourceName.substring(originResourceName.lastIndexOf('.')); //if the name without suffix then add it ,else use the origin name String nameWithSuffix = name; @@ -746,8 +739,7 @@ public class ResourcesService extends BaseService { logger.info("resource hdfs path is {} ", hdfsFileName); HadoopUtils.getInstance().copyHdfsToLocal(hdfsFileName, localFileName, false, true); - org.springframework.core.io.Resource file = org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(localFileName); - return file; + return org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(localFileName); } @@ -766,7 +758,7 @@ public class ResourcesService extends BaseService { } List resourceList = resourcesMapper.queryResourceExceptUserId(userId); List list ; - if (resourceList != null && resourceList.size() > 0) { + if (CollectionUtils.isNotEmpty(resourceList)) { Set resourceSet = new HashSet<>(resourceList); List authedResourceList = resourcesMapper.queryAuthorizedResourceList(userId); @@ -801,7 +793,7 @@ public class ResourcesService extends BaseService { List udfFuncList = udfFunctionMapper.queryUdfFuncExceptUserId(userId); List resultList = new ArrayList<>(); Set udfFuncSet = null; - if (udfFuncList != null && udfFuncList.size() > 0) { + if (CollectionUtils.isNotEmpty(udfFuncList)) { udfFuncSet = new HashSet<>(udfFuncList); List authedUDFFuncList = udfFunctionMapper.queryAuthedUdfFunc(userId); @@ -897,10 +889,9 @@ public class ResourcesService extends BaseService { */ private void getAuthorizedResourceList(Set resourceSet, List authedResourceList) { Set authedResourceSet = null; - if (authedResourceList != null && authedResourceList.size() > 0) { + if (CollectionUtils.isNotEmpty(authedResourceList)) { authedResourceSet = new HashSet<>(authedResourceList); resourceSet.removeAll(authedResourceSet); - } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index ca928c94d5..725188d6b0 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -71,7 +71,7 @@ public class PropertyUtils { * * @return judge whether resource upload startup */ - public static Boolean getResUploadStartupState(){ + public static boolean getResUploadStartupState(){ String resUploadStartupType = PropertyUtils.getString(Constants.RES_UPLOAD_STARTUP_TYPE); ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); return resUploadType == ResUploadType.HDFS || resUploadType == ResUploadType.S3; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java index aaf5129c02..01082414a9 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java @@ -138,7 +138,7 @@ public class ResourceMapperTest { resource.setCreateTime(new Date()); //update int update = resourceMapper.updateById(resource); - Assert.assertEquals(update, 1); + Assert.assertEquals(1, update); resourceMapper.deleteById(resource.getId()); } @@ -149,7 +149,7 @@ public class ResourceMapperTest { public void testDelete(){ Resource resourceMap = insertOne(); int delete = resourceMapper.deleteById(resourceMap.getId()); - Assert.assertEquals(delete, 1); + Assert.assertEquals(1, delete); } /** @@ -236,8 +236,8 @@ public class ResourceMapperTest { resourceUserMapper.deleteById(resourcesUser.getId()); resourceMapper.deleteById(resource.getId()); - Assert.assertEquals(resources.size(), 0); - Assert.assertNotEquals(resources1.size(), 0); + Assert.assertEquals(0, resources.size()); + Assert.assertNotEquals(0, resources1.size()); } @@ -251,7 +251,7 @@ public class ResourceMapperTest { List resources = resourceMapper.queryAuthorizedResourceList(resource.getUserId()); resourceMapper.deleteById(resource.getId()); - Assert.assertEquals(resources.size(), 0); + Assert.assertEquals(0, resources.size()); } /** @@ -293,7 +293,7 @@ public class ResourceMapperTest { ); - Assert.assertEquals(resource1, "ut tenant code for resource"); + Assert.assertEquals("ut tenant code for resource", resource1); resourceMapper.deleteById(resource.getId()); } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java index 233e88c5dd..2434a8c284 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java @@ -62,7 +62,7 @@ public class ResourceUserMapperTest { queue.setCreateTime(new Date()); //update int update = resourceUserMapper.updateById(queue); - Assert.assertEquals(update, 1); + Assert.assertEquals(1, update); resourceUserMapper.deleteById(queue.getId()); } @@ -73,7 +73,7 @@ public class ResourceUserMapperTest { public void testDelete(){ ResourcesUser queue = insertOne(); int delete = resourceUserMapper.deleteById(queue.getId()); - Assert.assertEquals(delete, 1); + Assert.assertEquals(1, delete); } /** diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java index ca304646e4..2e3954f4bc 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java @@ -32,9 +32,9 @@ import java.util.concurrent.*; */ public class ResponseFuture { - private final static Logger LOGGER = LoggerFactory.getLogger(ResponseFuture.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ResponseFuture.class); - private final static ConcurrentHashMap FUTURE_TABLE = new ConcurrentHashMap<>(256); + private static final ConcurrentHashMap FUTURE_TABLE = new ConcurrentHashMap<>(256); /** * request unique identification @@ -63,11 +63,11 @@ public class ResponseFuture { /** * response command */ - private volatile Command responseCommand; + private Command responseCommand; private volatile boolean sendOk = true; - private volatile Throwable cause; + private Throwable cause; public ResponseFuture(long opaque, long timeoutMillis, InvokeCallback invokeCallback, ReleaseSemaphore releaseSemaphore) { this.opaque = opaque; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/RemoveZKNode.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/RemoveZKNode.java index 5550e750b5..caec6e78a8 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/RemoveZKNode.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/RemoveZKNode.java @@ -28,7 +28,7 @@ import org.springframework.context.annotation.ComponentScan; @ComponentScan("org.apache.dolphinscheduler") public class RemoveZKNode implements CommandLineRunner { - private static Integer ARGS_LENGTH = 1; + private static final Integer ARGS_LENGTH = 1; private static final Logger logger = LoggerFactory.getLogger(RemoveZKNode.class); From ca6872a9a7f51f22142786a3f1c1418e4eb8b99b Mon Sep 17 00:00:00 2001 From: Rubik-W <39549317+Rubik-W@users.noreply.github.com> Date: Fri, 20 Mar 2020 21:27:52 +0800 Subject: [PATCH 06/22] =?UTF-8?q?fix=EF=BC=9ADepend=20check=20bug=20(#2229?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix：#1787 * fix：#1787 --- .../server/master/runner/MasterExecThread.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 26457a386a..3ef795938f 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -674,9 +674,9 @@ public class MasterExecThread implements Runnable { TaskNode taskNode = dag.getNode(taskName); List depNameList = taskNode.getDepList(); for(String depsNode : depNameList ){ - - if(forbiddenTaskList.containsKey(depsNode) || - skipTaskNodeList.containsKey(depsNode)){ + if(!dag.containsNode(depsNode) + || forbiddenTaskList.containsKey(depsNode) + || skipTaskNodeList.containsKey(depsNode)){ continue; } // dependencies must be fully completed From 8085e9310b6490dde3e7d1894a28326f3b157594 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sat, 21 Mar 2020 15:16:06 +0800 Subject: [PATCH 07/22] Adapting partial code(file name start with Q) to the sonar cloud rule (#2242) --- .../api/service/QueueService.java | 26 +++++++++---------- .../dao/mapper/QueueMapperTest.java | 4 +-- .../service/quartz/QuartzExecutors.java | 5 ++-- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java index 862c895c92..cba1b5f2bb 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.dao.entity.Queue; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.QueueMapper; @@ -43,7 +44,7 @@ import java.util.Map; @Service public class QueueService extends BaseService { - private static final Logger logger = LoggerFactory.getLogger(TenantService.class); + private static final Logger logger = LoggerFactory.getLogger(QueueService.class); @Autowired private QueueMapper queueMapper; @@ -186,19 +187,16 @@ public class QueueService extends BaseService { } // check queue name is exist - if (!queueName.equals(queueObj.getQueueName())) { - if (checkQueueNameExist(queueName)) { - putMsg(result, Status.QUEUE_NAME_EXIST, queueName); - return result; - } + if (!queueName.equals(queueObj.getQueueName()) + && checkQueueNameExist(queueName)) { + putMsg(result, Status.QUEUE_NAME_EXIST, queueName); + return result; } // check queue value is exist - if (!queue.equals(queueObj.getQueue())) { - if (checkQueueExist(queue)) { - putMsg(result, Status.QUEUE_VALUE_EXIST, queue); - return result; - } + if (!queue.equals(queueObj.getQueue()) && checkQueueExist(queue)) { + putMsg(result, Status.QUEUE_VALUE_EXIST, queue); + return result; } // check old queue using by any user @@ -267,7 +265,7 @@ public class QueueService extends BaseService { * @return true if the queue not exists, otherwise return false */ private boolean checkQueueExist(String queue) { - return queueMapper.queryAllQueueList(queue, null).size() > 0; + return CollectionUtils.isNotEmpty(queueMapper.queryAllQueueList(queue, null)); } /** @@ -278,7 +276,7 @@ public class QueueService extends BaseService { * @return true if the queue name not exists, otherwise return false */ private boolean checkQueueNameExist(String queueName) { - return queueMapper.queryAllQueueList(null, queueName).size() > 0; + return CollectionUtils.isNotEmpty(queueMapper.queryAllQueueList(null, queueName)); } /** @@ -290,7 +288,7 @@ public class QueueService extends BaseService { * @return true if need to update user */ private boolean checkIfQueueIsInUsing (String oldQueue, String newQueue) { - return !oldQueue.equals(newQueue) && userMapper.queryUserListByQueue(oldQueue).size() > 0; + return !oldQueue.equals(newQueue) && CollectionUtils.isNotEmpty(userMapper.queryUserListByQueue(oldQueue)); } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java index 30d2be03e0..ad3b05a048 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java @@ -64,7 +64,7 @@ public class QueueMapperTest { queue.setCreateTime(new Date()); //update int update = queueMapper.updateById(queue); - Assert.assertEquals(update, 1); + Assert.assertEquals(1, update); queueMapper.deleteById(queue.getId()); } @@ -75,7 +75,7 @@ public class QueueMapperTest { public void testDelete(){ Queue queue = insertOne(); int delete = queueMapper.deleteById(queue.getId()); - Assert.assertEquals(delete, 1); + Assert.assertEquals(1, delete); } /** diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java index 60cdb1dd97..30e7c52b19 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java @@ -70,9 +70,10 @@ public class QuartzExecutors { synchronized (QuartzExecutors.class) { // when more than two threads run into the first null check same time, to avoid instanced more than one time, it needs to be checked again. if (INSTANCE == null) { - INSTANCE = new QuartzExecutors(); + QuartzExecutors quartzExecutors = new QuartzExecutors(); //finish QuartzExecutors init - INSTANCE.init(); + quartzExecutors.init(); + INSTANCE = quartzExecutors; } } } From cddf803ba071bef565931c18e4e79e2f40035c45 Mon Sep 17 00:00:00 2001 From: khadgarmage Date: Sat, 21 Mar 2020 20:23:13 +0800 Subject: [PATCH 08/22] license and checkoutv2 fix (#2261) --- .github/workflows/ci_ut.yml | 10 +++--- .../DOLPHIN/1.2.1/package/scripts/params.py | 4 +-- docker/docker-compose.yml | 15 ++++++++ dolphinscheduler-remote/pom.xml | 17 ++++++++- dolphinscheduler-service/pom.xml | 17 ++++++++- .../src/lib/external/config.js | 18 +++++++++- dolphinscheduler-ui/src/lib/external/email.js | 16 +++++++++ e2e/pom.xml | 16 +++++++++ e2e/testng.xml | 16 +++++++++ pom.xml | 36 +++++++++---------- 10 files changed, 136 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 12f7c04ed6..95dfed8119 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -48,6 +48,11 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 + - name: Git fetch unshallow + run: | + git fetch --unshallow + git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + git fetch origin - name: Compile run: | export MAVEN_OPTS='-Dmaven.repo.local=.m2/repository -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit -Xmx3g' @@ -56,11 +61,6 @@ jobs: if: github.event_name == 'pull_request' run: | CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash) - - name: Git fetch unshallow - run: | - git fetch --unshallow - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - git fetch origin - name: Run SonarCloud Analysis run: > mvn verify --batch-mode diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py b/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py index 3780f6c27e..93b3249614 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py +++ b/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py @@ -7,16 +7,16 @@ to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE_2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - """ + import sys from resource_management import * from resource_management.core.logger import Logger diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index de5908583c..7e9c4e57cb 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. version: '2' services: zookeeper: diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml index 39c7c6a7c0..0968e610bc 100644 --- a/dolphinscheduler-remote/pom.xml +++ b/dolphinscheduler-remote/pom.xml @@ -1,5 +1,20 @@ - + diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml index 03bb94dc29..c150e834b9 100644 --- a/dolphinscheduler-service/pom.xml +++ b/dolphinscheduler-service/pom.xml @@ -1,5 +1,20 @@ - + diff --git a/dolphinscheduler-ui/src/lib/external/config.js b/dolphinscheduler-ui/src/lib/external/config.js index ac47c1ac7c..497540be7b 100644 --- a/dolphinscheduler-ui/src/lib/external/config.js +++ b/dolphinscheduler-ui/src/lib/external/config.js @@ -1,7 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /** * project external config */ export default { // task record switch recordSwitch:false -} \ No newline at end of file +} diff --git a/dolphinscheduler-ui/src/lib/external/email.js b/dolphinscheduler-ui/src/lib/external/email.js index 9398c10127..6b10efbf0f 100644 --- a/dolphinscheduler-ui/src/lib/external/email.js +++ b/dolphinscheduler-ui/src/lib/external/email.js @@ -1 +1,17 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ export default ["test@analysys.com.cn"] diff --git a/e2e/pom.xml b/e2e/pom.xml index ea67c0ca16..a5bdf69b41 100644 --- a/e2e/pom.xml +++ b/e2e/pom.xml @@ -1,4 +1,20 @@ + diff --git a/e2e/testng.xml b/e2e/testng.xml index 56c9649135..539d383a5a 100644 --- a/e2e/testng.xml +++ b/e2e/testng.xml @@ -1,4 +1,20 @@ + diff --git a/pom.xml b/pom.xml index dcb712a578..d11b3bf1f6 100644 --- a/pom.xml +++ b/pom.xml @@ -778,25 +778,23 @@ apache-rat-plugin ${apache.rat.version} - - **/*.java - **/dolphinscheduler-ui/src/**/*.scss - **/dolphinscheduler-ui/src/**/*.css - **/dolphinscheduler-ui/src/**/*.vue - **/dolphinscheduler-ui/src/**/*.js - **/dolphinscheduler-ui/src/**/*.html - - **/dolphinscheduler-ui/src/lib/** - **/dolphinscheduler-ui/src/js/module/util/cookie.js - **/dolphinscheduler-ui/src/font/awesome/font-awesome.css - **/dolphinscheduler-ui/src/sass/common/_animation.scss - **/dolphinscheduler-ui/src/sass/common/_normalize.scss - **/dolphinscheduler-ui/src/combo/1.0.0/es5.js - **/dolphinscheduler-ui/src/combo/1.0.0/base.css - **/dolphinscheduler-ui/src/view/common/outro.inc - **/dolphinscheduler-ui/src/view/common/meta.inc - **/dolphinscheduler-ui/src/combo/1.0.0/3rd.css + **/node_modules/** + **/node/** + **/dist/** + **/licenses/** + **/src/sass/common/_animation.scss + **/src/sass/common/_normalize.scss + .github/** + sql/soft_version + **/*.json + + **/*.md + **/*.MD + **/*.txt + **/docs/** + **/*.babelrc + **/*.eslintrc true @@ -888,4 +886,4 @@ dolphinscheduler-service - \ No newline at end of file + From 1fe5b31468aa62f8477bc17044d7519403f8ee12 Mon Sep 17 00:00:00 2001 From: khadgarmage Date: Sat, 21 Mar 2020 23:34:36 +0800 Subject: [PATCH 09/22] Fix checkoutv2 push trigger on dev (#2263) * push trigger on dev --- .github/workflows/ci_ut.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 95dfed8119..8d75bae163 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -15,7 +15,11 @@ # limitations under the License. # -on: ["pull_request", "push"] +on: + pull_request: + push: + branches: + - dev env: DOCKER_DIR: ./docker LOG_DIR: /tmp/dolphinscheduler @@ -58,7 +62,6 @@ jobs: export MAVEN_OPTS='-Dmaven.repo.local=.m2/repository -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit -Xmx3g' mvn test -B -Dmaven.test.skip=false - name: Upload coverage report to codecov - if: github.event_name == 'pull_request' run: | CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash) - name: Run SonarCloud Analysis From 34e2f67e0e23123ec5cede7d58a5c0a8b8a8afbf Mon Sep 17 00:00:00 2001 From: liwenhe1993 <32166572+liwenhe1993@users.noreply.github.com> Date: Sun, 22 Mar 2020 22:07:59 +0800 Subject: [PATCH 10/22] when running windows bat script, the handle is invalid in win7 (#2269) --- .../common/utils/process/ProcessImplForWin32.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java index 9f2716a096..bef5bb0ae2 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java @@ -32,6 +32,7 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; +import static com.sun.jna.platform.win32.WinBase.INVALID_HANDLE_VALUE; import static com.sun.jna.platform.win32.WinBase.STILL_ACTIVE; import static java.util.Objects.requireNonNull; @@ -742,7 +743,9 @@ public class ProcessImplForWin32 extends Process { } private static void closeHandle(WinNT.HANDLE handle) { - Kernel32Util.closeHandle(handle); + if (!handle.equals(INVALID_HANDLE_VALUE)) { + Kernel32Util.closeHandle(handle); + } } /** From 85f55cd5f927050fac10361a0d705344e2730b93 Mon Sep 17 00:00:00 2001 From: songquanhe-gitstudy <1172417734@qq.com> Date: Sun, 22 Mar 2020 22:32:20 +0800 Subject: [PATCH 11/22] fix: No operations allowed after statement closed when running sql task (#2265) * sqlTask failed to run * get correct attribute value and logger content Co-authored-by: songqh --- .../common/utils/OSUtils.java | 6 ++--- .../server/worker/config/WorkerConfig.java | 2 +- .../server/worker/task/sql/SqlTask.java | 24 +++++++++---------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index b011c0bc4e..a0fea8d33c 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -425,13 +425,13 @@ public class OSUtils { * @return check memory and cpu usage */ public static Boolean checkResource(double systemCpuLoad, double systemReservedMemory){ - // judging usage + // system load average double loadAverage = OSUtils.loadAverage(); - // + // system available physical memory double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); if(loadAverage > systemCpuLoad || availablePhysicalMemorySize < systemReservedMemory){ - logger.warn("load or availablePhysicalMemorySize(G) is too high, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize , loadAverage); + logger.warn("load is too high or availablePhysicalMemorySize(G) is too low, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize , loadAverage); return false; }else{ return true; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java index c4d4b61af5..aaaf5c7805 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java @@ -34,7 +34,7 @@ public class WorkerConfig { @Value("${worker.max.cpuload.avg}") private int workerMaxCpuloadAvg; - @Value("${master.reserved.memory}") + @Value("${worker.reserved.memory}") private double workerReservedMemory; public int getWorkerExecThreads() { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index 71ab56b528..f7070b1d38 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -363,20 +363,20 @@ public class SqlTask extends AbstractTask { // is the timeout set boolean timeoutFlag = taskProps.getTaskTimeoutStrategy() == TaskTimeoutStrategy.FAILED || taskProps.getTaskTimeoutStrategy() == TaskTimeoutStrategy.WARNFAILED; - try (PreparedStatement stmt = connection.prepareStatement(sqlBinds.getSql())) { - if(timeoutFlag){ - stmt.setQueryTimeout(taskProps.getTaskTimeout()); - } - Map params = sqlBinds.getParamsMap(); - if(params != null) { - for (Map.Entry entry : params.entrySet()) { - Property prop = entry.getValue(); - ParameterUtils.setInParameter(entry.getKey(), stmt, prop.getType(), prop.getValue()); - } + // prepare statement + PreparedStatement stmt = connection.prepareStatement(sqlBinds.getSql()); + if(timeoutFlag){ + stmt.setQueryTimeout(taskProps.getTaskTimeout()); + } + Map params = sqlBinds.getParamsMap(); + if(params != null) { + for (Map.Entry entry : params.entrySet()) { + Property prop = entry.getValue(); + ParameterUtils.setInParameter(entry.getKey(), stmt, prop.getType(), prop.getValue()); } - logger.info("prepare statement replace sql : {} ", stmt); - return stmt; } + logger.info("prepare statement replace sql : {} ", stmt); + return stmt; } /** From 3ced3d594b5318395865738a1fd130f49d5902aa Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Mon, 23 Mar 2020 10:10:31 +0800 Subject: [PATCH 12/22] datax add custom and add branch flow node verification (#2218) * Change DOM label * Change name to lowercase * Limit customization file content to no more than 3000 lines * dd branch flow node verification * datax * datax add custom --- .../js/conf/home/pages/dag/_source/dag.scss | 2 +- .../js/conf/home/pages/dag/_source/dag.vue | 91 ++-- .../pages/dag/_source/formModel/formModel.vue | 4 +- .../formModel/tasks/_source/nodeStatus.vue | 4 - .../dag/_source/formModel/tasks/datax.vue | 389 ++++++++++++------ .../module/components/transfer/resource.vue | 4 +- .../src/js/module/i18n/locale/en_US.js | 5 +- .../src/js/module/i18n/locale/zh_CN.js | 5 +- 8 files changed, 325 insertions(+), 179 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss index 6d97856960..9973750d98 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss @@ -135,7 +135,7 @@ width: 36px; height: 36px; float: left; - margin-bottom: 11px; + margin-bottom: 8px; border-radius: 3px; .disabled { .icos { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index a1ccd39260..8628fdb8ef 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -326,45 +326,62 @@ * Storage interface */ _save (sourceType) { - return new Promise((resolve, reject) => { - this.spinnerLoading = true - // Storage store - Dag.saveStore().then(res => { - if (this.urlParam.id) { - /** - * Edit - * @param saveInstanceEditDAGChart => Process instance editing - * @param saveEditDAGChart => Process definition editing - */ - this[this.type === 'instance' ? 'updateInstance' : 'updateDefinition'](this.urlParam.id).then(res => { - this.$message.success(res.msg) - this.spinnerLoading = false - resolve() - }).catch(e => { - this.$message.error(e.msg || '') - this.spinnerLoading = false - reject(e) - }) - } else { - // New - this.saveDAGchart().then(res => { - this.$message.success(res.msg) - this.spinnerLoading = false - // source @/conf/home/pages/dag/_source/editAffirmModel/index.js - if (sourceType !== 'affirm') { - // Jump process definition - this.$router.push({ name: 'projects-definition-list' }) - } - resolve() - }).catch(e => { - this.$message.error(e.msg || '') - this.setName('') - this.spinnerLoading = false - reject(e) - }) - } + if(this._verifConditions()) { + return new Promise((resolve, reject) => { + this.spinnerLoading = true + // Storage store + Dag.saveStore().then(res => { + if (this.urlParam.id) { + /** + * Edit + * @param saveInstanceEditDAGChart => Process instance editing + * @param saveEditDAGChart => Process definition editing + */ + this[this.type === 'instance' ? 'updateInstance' : 'updateDefinition'](this.urlParam.id).then(res => { + this.$message.success(res.msg) + this.spinnerLoading = false + resolve() + }).catch(e => { + this.$message.error(e.msg || '') + this.spinnerLoading = false + reject(e) + }) + } else { + // New + this.saveDAGchart().then(res => { + this.$message.success(res.msg) + this.spinnerLoading = false + // source @/conf/home/pages/dag/_source/editAffirmModel/index.js + if (sourceType !== 'affirm') { + // Jump process definition + this.$router.push({ name: 'projects-definition-list' }) + } + resolve() + }).catch(e => { + this.$message.error(e.msg || '') + this.setName('') + this.spinnerLoading = false + reject(e) + }) + } + }) }) + } + }, + _verifConditions () { + let tasks = this.$store.state.dag.tasks + let bool = true + tasks.map(v=>{ + if(v.type == 'CONDITIONS' && (v.conditionResult.successNode[0] =='' || v.conditionResult.successNode[0] == null || v.conditionResult.failedNode[0] =='' || v.conditionResult.failedNode[0] == null)) { + bool = false + return false + } }) + if(!bool) { + this.$message.warning(`${i18n.$t('Successful branch flow and failed branch flow are required')}`) + return false + } + return true }, /** * Global parameter diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index cc1c8b6d6c..017f154b44 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -430,7 +430,7 @@ * return params */ _onParams (o) { - this.params = Object.assign(this.params, {}, o) + this.params = Object.assign({}, o) }, _onCacheParams (o) { @@ -470,7 +470,7 @@ this.$message.warning(`${i18n.$t('Please enter name (required)')}`) return false } - if (this.successBranch !='' && this.successBranch == this.failedBranch) { + if (this.successBranch !='' && this.successBranch !=null && this.successBranch == this.failedBranch) { this.$message.warning(`${i18n.$t('Cannot select the same node for successful branch flow and failed branch flow')}`) return false } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue index fa7ee89e98..0c3f7433a3 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue @@ -128,10 +128,6 @@ this.store.dispatch('dag/getProcessTasksList', { processDefinitionId: ids }).then(res => { resolve(['ALL'].concat(_.map(res, v => v.name))) }) - } else { - this.store.dispatch('dag/getTaskListDefIdAll', { processDefinitionIdList: ids }).then(res => { - resolve(res) - }) } }) }, diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue index 959610f95a..f1c9b757bd 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue @@ -17,90 +17,127 @@