Merge pull request #7 from apache/dev

update code from apache

.github/ISSUE_TEMPLATE/bug_report.md

@@ -9,6 +9,8 @@ assignees: ''
 
 *For better global communication, please give priority to using English description, thx! *
 
+*Please review https://dolphinscheduler.apache.org/en-us/docs/development/issue.html when describe an issue.*
+
 **Describe the bug**
 A clear and concise description of what the bug is.
 

.github/ISSUE_TEMPLATE/feature_request.md

@@ -9,6 +9,8 @@ assignees: ''
 
 *For better global communication, please give priority to using English description, thx! *
 
+*Please review https://dolphinscheduler.apache.org/en-us/docs/development/issue.html when describe an issue.*
+
 **Describe the feature**
 A clear and concise description of what the feature is.
 

.github/ISSUE_TEMPLATE/improvement_suggestion.md

@@ -9,6 +9,8 @@ assignees: ''
 
 *For better global communication, please give priority to using English description, thx! *
 
+*Please review https://dolphinscheduler.apache.org/en-us/docs/development/issue.html when describe an issue.*
+
 **Describe the question**
 A clear and concise description of what the improvement is.
 

.github/ISSUE_TEMPLATE/question.md

@@ -9,6 +9,8 @@ assignees: ''
 
 *For better global communication, please give priority to using English description, thx! *
 
+*Please review https://dolphinscheduler.apache.org/en-us/docs/development/issue.html when describe an issue.*
+
 **Describe the question**
 A clear and concise description of what the question is.
 

.github/ISSUE_TEMPLATE/test.md

@@ -9,6 +9,8 @@ assignees: ''
 
 *For better global communication, please give priority to using English description, thx! *
 
+*Please review https://dolphinscheduler.apache.org/en-us/docs/development/issue.html when describe an issue.*
+
 **Describe the question**
 A clear and concise description of what the test part is.
 

docker/build/conf/nginx/dolphinscheduler.conf

@@ -24,6 +24,9 @@ server {
         root   /opt/dolphinscheduler/ui;
         index  index.html index.html;
     }
+    location /dolphinscheduler/ui{
+        alias  /opt/dolphinscheduler/ui;
+    }
     location /dolphinscheduler {
         proxy_pass http://FRONTEND_API_SERVER_HOST:FRONTEND_API_SERVER_PORT;
         proxy_set_header Host $host;

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java

@@ -22,7 +22,6 @@ import org.springframework.boot.web.servlet.ServletComponentScan;
 import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.FilterType;
-import springfox.documentation.swagger2.annotations.EnableSwagger2;
 
 @SpringBootApplication
 @ServletComponentScan

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java

@@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.common.enums.Flag;
 import org.apache.dolphinscheduler.common.utils.ParameterUtils;
 import org.apache.dolphinscheduler.common.utils.StringUtils;
+import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
 import org.apache.dolphinscheduler.dao.entity.User;
 import io.swagger.annotations.*;
 import org.slf4j.Logger;
@@ -204,6 +205,39 @@ public class ProcessInstanceController extends BaseController {
         return returnDataList(result);
     }
 
+    /**
+     * query top n process instance order by running duration
+     *
+     * @param loginUser     login user
+     * @param projectName   project name
+     * @param size          number of process instance
+     * @param startTime     start time
+     * @param endTime       end time
+     * @return              list of process instance
+     */
+    @ApiOperation(value = "queryTopNLongestRunningProcessInstance", notes = "QUERY_TOPN_LONGEST_RUNNING_PROCESS_INSTANCE_NOTES")
+    @ApiImplicitParams({
+            @ApiImplicitParam(name = "size", value = "PROCESS_INSTANCE_SIZE", dataType = "Int", example = "10"),
+            @ApiImplicitParam(name = "startTime", value = "PROCESS_INSTANCE_START_TIME", dataType = "String"),
+            @ApiImplicitParam(name = "endTime", value = "PROCESS_INSTANCE_END_TIME", dataType = "String"),
+    })
+    @GetMapping(value = "/top-n")
+    @ResponseStatus(HttpStatus.OK)
+    @ApiException(QUERY_PROCESS_INSTANCE_BY_ID_ERROR)
+    public Result<ProcessInstance> queryTopNLongestRunningProcessInstance(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
+                                                         @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName,
+                                                         @RequestParam("size") Integer size,
+                                                         @RequestParam(value = "startTime",required = true) String startTime,
+                                                         @RequestParam(value = "endTime",required = true) String endTime
+
+    ) {
+        projectName=ParameterUtils.handleEscapes(projectName);
+        logger.info("query top {} SUCCESS process instance order by running time whprojectNameich started between {} and {} ,login user:{},project name:{}", size, startTime, endTime,
+                loginUser.getUserName(), projectName);
+        Map<String,Object> result=processInstanceService.queryTopNLongestRunningProcessInstance(loginUser, projectName, size, startTime, endTime);
+        return returnDataList(result);
+    }
+
     /**
      * delete process instance by id, at the same time,
      * delete task instance and their mapping relation data
@@ -220,9 +254,9 @@ public class ProcessInstanceController extends BaseController {
     @GetMapping(value = "/delete")
     @ResponseStatus(HttpStatus.OK)
     @ApiException(DELETE_PROCESS_INSTANCE_BY_ID_ERROR)
-    public Result deleteProcessInstanceById(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
+    public Result<ProcessInstance> deleteProcessInstanceById(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
-                                            @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName,
+                                                             @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName,
-                                            @RequestParam("processInstanceId") Integer processInstanceId
+                                                             @RequestParam("processInstanceId") Integer processInstanceId
     ) {
         logger.info("delete process instance by id, login user:{}, project name:{}, process instance id:{}",
                 loginUser.getUserName(), projectName, processInstanceId);

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java

@@ -52,7 +52,7 @@ public class SchedulerController extends BaseController {
     public static final String DEFAULT_WARNING_TYPE = "NONE";
     public static final String DEFAULT_NOTIFY_GROUP_ID = "1";
     public static final String DEFAULT_FAILURE_POLICY = "CONTINUE";
-
+    public static final String DEFAULT_PROCESS_INSTANCE_PRIORITY = "MEDIUM";
 
     @Autowired
     private SchedulerService schedulerService;
@@ -99,7 +99,7 @@ public class SchedulerController extends BaseController {
                                  @RequestParam(value = "receivers", required = false) String receivers,
                                  @RequestParam(value = "receiversCc", required = false) String receiversCc,
                                  @RequestParam(value = "workerGroup", required = false, defaultValue = "default") String workerGroup,
-                                 @RequestParam(value = "processInstancePriority", required = false) Priority processInstancePriority) throws IOException {
+                                 @RequestParam(value = "processInstancePriority", required = false, defaultValue = DEFAULT_PROCESS_INSTANCE_PRIORITY) Priority processInstancePriority) throws IOException {
         logger.info("login user {}, project name: {}, process name: {}, create schedule: {}, warning type: {}, warning group id: {}," +
                         "failure policy: {},receivers : {},receiversCc : {},processInstancePriority : {}, workGroupId:{}",
                 loginUser.getUserName(), projectName, processDefinitionId, schedule, warningType, warningGroupId,

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java

@@ -432,14 +432,34 @@ public class UsersController extends BaseController {
                                @RequestParam(value = "userPassword") String userPassword,
                                @RequestParam(value = "repeatPassword") String repeatPassword,
                                @RequestParam(value = "email") String email) throws Exception {
-        userName = userName.replaceAll("[\n|\r|\t]", "");
+        userName = ParameterUtils.handleEscapes(userName);
-        userPassword = userPassword.replaceAll("[\n|\r|\t]", "");
+        userPassword = ParameterUtils.handleEscapes(userPassword);
-        repeatPassword = repeatPassword.replaceAll("[\n|\r|\t]", "");
+        repeatPassword = ParameterUtils.handleEscapes(repeatPassword);
-        email = email.replaceAll("[\n|\r|\t]", "");
+        email = ParameterUtils.handleEscapes(email);
         logger.info("user self-register, userName: {}, userPassword {}, repeatPassword {}, eamil {}",
-                userName, userPassword, repeatPassword, email);
+                userName, Constants.PASSWORD_DEFAULT, Constants.PASSWORD_DEFAULT, email);
         Map<String, Object> result = usersService.registerUser(userName, userPassword, repeatPassword, email);
         return returnDataList(result);
     }
 
+    /**
+     * user activate
+     *
+     * @param userName       user name
+     */
+    @ApiOperation(value="activateUser",notes = "ACTIVATE_USER_NOTES")
+    @ApiImplicitParams({
+            @ApiImplicitParam(name = "userName", value = "USER_NAME", type = "String"),
+    })
+    @PostMapping("/activate")
+    @ResponseStatus(HttpStatus.OK)
+    @ApiException(UPDATE_USER_ERROR)
+    public Result<Object> activateUser(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
+                                       @RequestParam(value = "userName") String userName) {
+        userName = ParameterUtils.handleEscapes(userName);
+        logger.info("login user {}, activate user, userName: {}",
+                loginUser.getUserName(), userName);
+        Map<String, Object> result = usersService.activateUser(loginUser, userName);
+        return returnDataList(result);
+    }
 }

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageController.java

@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.api.controller;
+
+import org.apache.dolphinscheduler.api.service.WorkFlowLineageService;
+import org.apache.dolphinscheduler.api.utils.Result;
+import org.apache.dolphinscheduler.common.utils.ParameterUtils;
+import io.swagger.annotations.ApiParam;
+import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.*;
+import springfox.documentation.annotations.ApiIgnore;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.dolphinscheduler.api.enums.Status.QUERY_WORKFLOW_LINEAGE_ERROR;
+
+@RestController
+@RequestMapping("lineages/{projectId}")
+public class WorkFlowLineageController extends BaseController {
+    private static final Logger logger = LoggerFactory.getLogger(WorkFlowLineageController.class);
+
+    @Autowired
+    private WorkFlowLineageService workFlowLineageService;
+
+    @GetMapping(value="/list-name")
+    @ResponseStatus(HttpStatus.OK)
+    public Result<List<WorkFlowLineage>> queryWorkFlowLineageByName(@ApiIgnore @RequestParam(value = "searchVal", required = false) String searchVal, @ApiParam(name = "projectId", value = "PROJECT_ID", required = true) @PathVariable int projectId) {
+        try {
+            searchVal = ParameterUtils.handleEscapes(searchVal);
+            Map<String, Object> result = workFlowLineageService.queryWorkFlowLineageByName(searchVal,projectId);
+            return returnDataList(result);
+        } catch (Exception e){
+            logger.error(QUERY_WORKFLOW_LINEAGE_ERROR.getMsg(),e);
+            return error(QUERY_WORKFLOW_LINEAGE_ERROR.getCode(), QUERY_WORKFLOW_LINEAGE_ERROR.getMsg());
+        }
+    }
+
+    @GetMapping(value="/list-ids")
+    @ResponseStatus(HttpStatus.OK)
+    public Result<Map<String, Object>> queryWorkFlowLineageByIds(@ApiIgnore @RequestParam(value = "ids", required = false) String ids,@ApiParam(name = "projectId", value = "PROJECT_ID", required = true) @PathVariable int projectId) {
+
+        try {
+            ids = ParameterUtils.handleEscapes(ids);
+            Set<Integer> idsSet = new HashSet<>();
+            if(ids != null) {
+                String[] idsStr = ids.split(",");
+                for (String id : idsStr)
+                {
+                    idsSet.add(Integer.parseInt(id));
+                }
+            }
+
+            Map<String, Object> result = workFlowLineageService.queryWorkFlowLineageByIds(idsSet, projectId);
+            return returnDataList(result);
+        } catch (Exception e){
+            logger.error(QUERY_WORKFLOW_LINEAGE_ERROR.getMsg(),e);
+            return error(QUERY_WORKFLOW_LINEAGE_ERROR.getCode(), QUERY_WORKFLOW_LINEAGE_ERROR.getMsg());
+        }
+    }
+}

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java

@@ -62,7 +62,7 @@ public class TaskCountDto {
                 case SUBMITTED_SUCCESS:
                     submittedSuccess += taskInstanceStateCount.getCount();
                     break;
-                case RUNNING_EXEUTION:
+                case RUNNING_EXECUTION:
                     runningExeution += taskInstanceStateCount.getCount();
                     break;
                 case READY_PAUSE:
@@ -99,7 +99,7 @@ public class TaskCountDto {
         }
         this.taskCountDtos = new ArrayList<>();
         this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.SUBMITTED_SUCCESS, submittedSuccess));
-        this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.RUNNING_EXEUTION, runningExeution));
+        this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.RUNNING_EXECUTION, runningExeution));
         this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.READY_PAUSE, readyPause));
         this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.PAUSE, pause));
         this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.READY_STOP, readyStop));

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java

@@ -168,15 +168,15 @@ public enum Status {
     PREVIEW_SCHEDULE_ERROR(10139,"preview schedule error", "预览调度配置错误"),
     PARSE_TO_CRON_EXPRESSION_ERROR(10140,"parse cron to cron expression error", "解析调度表达式错误"),
     SCHEDULE_START_TIME_END_TIME_SAME(10141,"The start time must not be the same as the end", "开始时间不能和结束时间一样"),
-    DELETE_TENANT_BY_ID_FAIL(10142,"delete tenant by id fail, for there are {0} process instances in executing using it", "删除租户失败，有[{0}]个运行中的工作流实例正在使用"),
+    DELETE_TENANT_BY_ID_FAIL(100142,"delete tenant by id fail, for there are {0} process instances in executing using it", "删除租户失败，有[{0}]个运行中的工作流实例正在使用"),
-    DELETE_TENANT_BY_ID_FAIL_DEFINES(10143,"delete tenant by id fail, for there are {0} process definitions using it", "删除租户失败，有[{0}]个工作流定义正在使用"),
+    DELETE_TENANT_BY_ID_FAIL_DEFINES(100143,"delete tenant by id fail, for there are {0} process definitions using it", "删除租户失败，有[{0}]个工作流定义正在使用"),
-    DELETE_TENANT_BY_ID_FAIL_USERS(10144,"delete tenant by id fail, for there are {0} users using it", "删除租户失败，有[{0}]个用户正在使用"),
+    DELETE_TENANT_BY_ID_FAIL_USERS(100144,"delete tenant by id fail, for there are {0} users using it", "删除租户失败，有[{0}]个用户正在使用"),
-    DELETE_WORKER_GROUP_BY_ID_FAIL(10145,"delete worker group by id fail, for there are {0} process instances in executing using it", "删除Worker分组失败，有[{0}]个运行中的工作流实例正在使用"),
+    DELETE_WORKER_GROUP_BY_ID_FAIL(100145,"delete worker group by id fail, for there are {0} process instances in executing using it", "删除Worker分组失败，有[{0}]个运行中的工作流实例正在使用"),
-    QUERY_WORKER_GROUP_FAIL(10146,"query worker group fail ", "查询worker分组失败"),
+    QUERY_WORKER_GROUP_FAIL(100146,"query worker group fail ", "查询worker分组失败"),
-    DELETE_WORKER_GROUP_FAIL(10147,"delete worker group fail ", "删除worker分组失败"),
+    DELETE_WORKER_GROUP_FAIL(100147,"delete worker group fail ", "删除worker分组失败"),
+    QUERY_WORKFLOW_LINEAGE_ERROR(10143,"query workflow lineage error", "查询血缘失败"),
     COPY_PROCESS_DEFINITION_ERROR(10148,"copy process definition error", "复制工作流错误"),
     USER_DISABLED(10149,"The current user is disabled", "当前用户已停用"),
-
     UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"),
     UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"),
     RESOURCE_NOT_EXIST(20004, "resource not exist", "资源不存在"),
@@ -249,7 +249,8 @@ public enum Status {
 
 
     COMMAND_STATE_COUNT_ERROR(80001,"task instance state count error", "查询各状态任务实例数错误"),
-
+    NEGTIVE_SIZE_NUMBER_ERROR(80002,"query size number error","查询size错误"),
+    START_TIME_BIGGER_THAN_END_TIME_ERROR(80003,"start time bigger than end time error","开始时间在结束时间之后错误"),
     QUEUE_COUNT_ERROR(90001,"queue count error", "查询队列数据错误"),
 
     KERBEROS_STARTUP_STATE(100001,"get kerberos startup state error", "获取kerberos启动状态错误"),

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java

@@ -177,7 +177,7 @@ public class AlertGroupService extends BaseService{
      * @param id alert group id
      * @return delete result code
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public Map<String, Object> delAlertgroupById(User loginUser, int id) {
         Map<String, Object> result = new HashMap<>(5);
         result.put(Constants.STATUS, false);

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java

@@ -527,7 +527,7 @@ public class DataSourceService extends BaseService{
         parameterMap.put(Constants.DATABASE, database);
         parameterMap.put(Constants.JDBC_URL, jdbcUrl);
         parameterMap.put(Constants.USER, userName);
-        parameterMap.put(Constants.PASSWORD, password);
+        parameterMap.put(Constants.PASSWORD, CommonUtils.encodePassword(password));
         if (CommonUtils.getKerberosStartupState() &&
                 (type == DbType.HIVE || type == DbType.SPARK)){
             parameterMap.put(Constants.PRINCIPAL,principal);
@@ -600,7 +600,7 @@ public class DataSourceService extends BaseService{
      * @param datasourceId data source id
      * @return delete result code
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public Result delete(User loginUser, int datasourceId) {
         Result result = new Result();
         try {

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java

@@ -16,6 +16,9 @@
  */
 package org.apache.dolphinscheduler.api.service;
 
+import java.nio.charset.StandardCharsets;
+import javax.annotation.PreDestroy;
+import org.apache.commons.lang.ArrayUtils;
 import org.apache.dolphinscheduler.api.enums.Status;
 import org.apache.dolphinscheduler.api.utils.Result;
 import org.apache.dolphinscheduler.common.Constants;
@@ -29,8 +32,6 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import javax.annotation.PreDestroy;
-
 /**
  * log service
  */
@@ -39,17 +40,19 @@ public class LoggerService {
 
   private static final Logger logger = LoggerFactory.getLogger(LoggerService.class);
 
+  private static final String LOG_HEAD_FORMAT = "[LOG-PATH]: %s, [HOST]:  %s%s";
+
   @Autowired
   private ProcessService processService;
 
   private final LogClientService logClient;
 
-  public LoggerService(){
+  public LoggerService() {
     logClient = new LogClientService();
   }
 
   @PreDestroy
-  public void close(){
+  public void close() {
     logClient.close();
   }
 
@@ -65,24 +68,34 @@ public class LoggerService {
 
     TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId);
 
-    if (taskInstance == null || StringUtils.isBlank(taskInstance.getHost())){
+    if (taskInstance == null || StringUtils.isBlank(taskInstance.getHost())) {
-      return new Result(Status.TASK_INSTANCE_NOT_FOUND.getCode(), Status.TASK_INSTANCE_NOT_FOUND.getMsg());
+      return Result.error(Status.TASK_INSTANCE_NOT_FOUND);
     }
 
     String host = getHost(taskInstance.getHost());
 
     Result result = new Result(Status.SUCCESS.getCode(), Status.SUCCESS.getMsg());
 
-    logger.info("log host : {} , logPath : {} , logServer port : {}",host,taskInstance.getLogPath(),Constants.RPC_PORT);
+    logger.info("log host : {} , logPath : {} , logServer port : {}", host, taskInstance.getLogPath(),
+            Constants.RPC_PORT);
+
+    StringBuilder log = new StringBuilder();
+    if (skipLineNum == 0) {
+      String head = String.format(LOG_HEAD_FORMAT,
+          taskInstance.getLogPath(),
+          host,
+          Constants.SYSTEM_LINE_SEPARATOR);
+      log.append(head);
+    }
+
+    log.append(logClient
+        .rollViewLog(host, Constants.RPC_PORT, taskInstance.getLogPath(), skipLineNum, limit));
 
-    String log = logClient.rollViewLog(host, Constants.RPC_PORT, taskInstance.getLogPath(),skipLineNum,limit);
     result.setData(log);
     return result;
   }
 
 
-
-
   /**
    * get log size
    *
@@ -91,22 +104,27 @@ public class LoggerService {
    */
   public byte[] getLogBytes(int taskInstId) {
     TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId);
-    if (taskInstance == null || StringUtils.isBlank(taskInstance.getHost())){
+    if (taskInstance == null || StringUtils.isBlank(taskInstance.getHost())) {
       throw new RuntimeException("task instance is null or host is null");
     }
     String host = getHost(taskInstance.getHost());
-
+    byte[] head = String.format(LOG_HEAD_FORMAT,
-    return logClient.getLogBytes(host, Constants.RPC_PORT, taskInstance.getLogPath());
+        taskInstance.getLogPath(),
+        host,
+        Constants.SYSTEM_LINE_SEPARATOR).getBytes(StandardCharsets.UTF_8);
+    return ArrayUtils.addAll(head,
+        logClient.getLogBytes(host, Constants.RPC_PORT, taskInstance.getLogPath()));
   }
 
 
   /**
    * get host
+   *
    * @param address address
    * @return old version return true ,otherwise return false
    */
-  private String getHost(String address){
+  private String getHost(String address) {
-    if (Host.isOldVersion(address)){
+    if (Host.isOldVersion(address)) {
       return address;
     }
     return Host.of(address).getIp();

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java

@@ -438,7 +438,7 @@ public class ProcessDefinitionService extends BaseDAGService {
      * @param processDefinitionId process definition id
      * @return delete result code
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public Map<String, Object> deleteProcessDefinitionById(User loginUser, String projectName, Integer processDefinitionId) {
 
         Map<String, Object> result = new HashMap<>(5);
@@ -504,7 +504,7 @@ public class ProcessDefinitionService extends BaseDAGService {
      * @param releaseState release state
      * @return release result code
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public Map<String, Object> releaseProcessDefinition(User loginUser, String projectName, int id, int releaseState) {
         HashMap<String, Object> result = new HashMap<>();
         Project project = projectMapper.queryByName(projectName);
@@ -751,7 +751,7 @@ public class ProcessDefinitionService extends BaseDAGService {
      * @param currentProjectName current project name
      * @return import process
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public Map<String, Object> importProcessDefinition(User loginUser, MultipartFile file, String currentProjectName) {
         Map<String, Object> result = new HashMap<>(5);
         String processMetaJson = FileUtils.file2String(file);

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java

@@ -98,6 +98,53 @@ public class ProcessInstanceService extends BaseDAGService {
     @Autowired
     UsersService usersService;
 
+    /**
+     * return top n SUCCESS process instance order by running time which started between startTime and endTime
+     * @param loginUser
+     * @param projectName
+     * @param size
+     * @param startTime
+     * @param endTime
+     * @return
+     */
+    public Map<String, Object> queryTopNLongestRunningProcessInstance(User loginUser, String projectName, int size, String startTime, String endTime) {
+        Map<String, Object> result = new HashMap<>();
+
+        Project project = projectMapper.queryByName(projectName);
+        Map<String, Object> checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName);
+        Status resultEnum = (Status) checkResult.get(Constants.STATUS);
+        if (resultEnum != Status.SUCCESS) {
+            return checkResult;
+        }
+
+        if (0 > size) {
+            putMsg(result, Status.NEGTIVE_SIZE_NUMBER_ERROR, size);
+            return result;
+        }
+        if (Objects.isNull(startTime)) {
+            putMsg(result, Status.DATA_IS_NULL, Constants.START_TIME);
+            return result;
+        }
+        Date start = DateUtils.stringToDate(startTime);
+        if (Objects.isNull(endTime)) {
+            putMsg(result, Status.DATA_IS_NULL, Constants.END_TIME);
+            return result;
+        }
+        Date end = DateUtils.stringToDate(endTime);
+        if(start == null || end == null) {
+            putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "startDate,endDate");
+            return result;
+        }
+        if (start.getTime() > end.getTime()) {
+            putMsg(result, Status.START_TIME_BIGGER_THAN_END_TIME_ERROR, startTime, endTime);
+            return result;
+        }
+
+        List<ProcessInstance> processInstances = processInstanceMapper.queryTopNProcessInstance(size, start, end, ExecutionStatus.SUCCESS);
+        result.put(DATA_LIST, processInstances);
+        putMsg(result, Status.SUCCESS);
+        return result;
+    }
     /**
      * query process instance by id
      *
@@ -466,7 +513,7 @@ public class ProcessInstanceService extends BaseDAGService {
      * @param processInstanceId process instance id
      * @return delete result code
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public Map<String, Object> deleteProcessInstanceById(User loginUser, String projectName, Integer processInstanceId) {
 
         Map<String, Object> result = new HashMap<>(5);

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java

@@ -86,7 +86,7 @@ public class ResourcesService extends BaseService {
      * @param currentDir current directory
      * @return create directory result
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public Result createDirectory(User loginUser,
                                  String name,
                                  String description,
@@ -160,7 +160,7 @@ public class ResourcesService extends BaseService {
      * @param currentDir current directory
      * @return create result code
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public Result createResource(User loginUser,
                                  String name,
                                  String desc,
@@ -288,7 +288,7 @@ public class ResourcesService extends BaseService {
      * @param type          resource type
      * @return  update result code
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public Result updateResource(User loginUser,
                                  int resourceId,
                                  String name,
@@ -827,7 +827,7 @@ public class ResourcesService extends BaseService {
      * @param content content
      * @return create result code
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public Result onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix, String desc, String content,int pid,String currentDirectory) {
         Result result = new Result();
         // if resource upload startup
@@ -889,7 +889,7 @@ public class ResourcesService extends BaseService {
      * @param content content
      * @return update result cod
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public Result updateResourceContent(int resourceId, String content) {
         Result result = new Result();
 

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java

@@ -47,7 +47,6 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
-import java.io.IOException;
 import java.text.ParseException;
 import java.util.*;
 
@@ -95,9 +94,8 @@ public class SchedulerService extends BaseService {
      * @param receiversCc receivers cc
      * @param workerGroup worker group
      * @return create result code
-     * @throws IOException ioexception
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public Map<String, Object> insertSchedule(User loginUser, String projectName,
                                               Integer processDefineId,
                                               String schedule,
@@ -107,7 +105,7 @@ public class SchedulerService extends BaseService {
                                               String receivers,
                                               String receiversCc,
                                               Priority processInstancePriority,
-                                              String workerGroup) throws IOException {
+                                              String workerGroup) {
 
         Map<String, Object> result = new HashMap<String, Object>(5);
 
@@ -192,9 +190,8 @@ public class SchedulerService extends BaseService {
      * @param receivers receivers
      * @param scheduleStatus schedule status
      * @return update result code
-     * @throws IOException ioexception
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public Map<String, Object> updateSchedule(User loginUser,
                                               String projectName,
                                               Integer id,
@@ -206,7 +203,7 @@ public class SchedulerService extends BaseService {
                                               String receiversCc,
                                               ReleaseState scheduleStatus,
                                               Priority processInstancePriority,
-                                              String workerGroup) throws IOException {
+                                              String workerGroup) {
         Map<String, Object> result = new HashMap<String, Object>(5);
 
         Project project = projectMapper.queryByName(projectName);
@@ -296,7 +293,7 @@ public class SchedulerService extends BaseService {
      * @param scheduleStatus  schedule status
      * @return publish result code
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public Map<String, Object> setScheduleState(User loginUser,
                                                 String projectName,
                                                 Integer id,

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SessionService.java

@@ -81,7 +81,7 @@ public class SessionService extends BaseService{
    * @param ip ip
    * @return session string
    */
-  @Transactional(rollbackFor = Exception.class)
+  @Transactional(rollbackFor = RuntimeException.class)
   public String createSession(User user, String ip) {
     Session session = null;
 

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java

@@ -300,7 +300,7 @@ public class UdfFuncService extends BaseService{
      * @param id udf function id
      * @return delete result code
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public Result delete(int id) {
         Result result = new Result();
         

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java

@@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.api.utils.CheckUtils;
 import org.apache.dolphinscheduler.api.utils.PageInfo;
 import org.apache.dolphinscheduler.api.utils.Result;
 import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.Flag;
 import org.apache.dolphinscheduler.common.enums.ResourceType;
 import org.apache.dolphinscheduler.common.enums.UserType;
 import org.apache.dolphinscheduler.common.utils.*;
@@ -138,14 +139,14 @@ public class UsersService extends BaseService {
 
     }
 
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public User createUser(String userName,
                                           String userPassword,
                                           String email,
                                           int tenantId,
                                           String phone,
                                           String queue,
-                                          int state) throws Exception {
+                                          int state) {
         User user = new User();
         Date now = new Date();
 
@@ -430,7 +431,7 @@ public class UsersService extends BaseService {
      * @param projectIds project id array
      * @return grant result code
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public Map<String, Object> grantProject(User loginUser, int userId, String projectIds) {
         Map<String, Object> result = new HashMap<>(5);
         result.put(Constants.STATUS, false);
@@ -480,7 +481,7 @@ public class UsersService extends BaseService {
      * @param resourceIds resource id array
      * @return grant result code
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public Map<String, Object> grantResources(User loginUser, int userId, String resourceIds) {
         Map<String, Object> result = new HashMap<>(5);
         //only admin can operate
@@ -577,7 +578,7 @@ public class UsersService extends BaseService {
      * @param udfIds udf id array
      * @return grant result code
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public Map<String, Object> grantUDFFunction(User loginUser, int userId, String udfIds) {
         Map<String, Object> result = new HashMap<>(5);
 
@@ -624,7 +625,7 @@ public class UsersService extends BaseService {
      * @param datasourceIds  data source id array
      * @return grant result code
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
     public Map<String, Object> grantDataSource(User loginUser, int userId, String datasourceIds) {
         Map<String, Object> result = new HashMap<>(5);
         result.put(Constants.STATUS, false);
@@ -919,9 +920,9 @@ public class UsersService extends BaseService {
      * @return register result code
      * @throws Exception exception
      */
-    @Transactional(rollbackFor = Exception.class)
+    @Transactional(rollbackFor = RuntimeException.class)
-    public Map<String, Object> registerUser(String userName, String userPassword, String repeatPassword, String email) throws Exception {
+    public Map<String, Object> registerUser(String userName, String userPassword, String repeatPassword, String email) {
-        Map<String, Object> result = new HashMap<>(5);
+        Map<String, Object> result = new HashMap<>();
 
         //check user params
         String msg = this.checkUserParams(userName, userPassword, email, "");
@@ -935,10 +936,51 @@ public class UsersService extends BaseService {
             putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "two passwords are not same");
             return result;
         }
-
+        User user = createUser(userName, userPassword, email, 1, "", "", Flag.NO.ordinal());
-        createUser(userName, userPassword, email, 1, "", "", 0);
         putMsg(result, Status.SUCCESS);
+        result.put(Constants.DATA_LIST, user);
         return result;
     }
 
+    /**
+     * activate user, only system admin have permission, change user state code 0 to 1
+     *
+     * @param loginUser login user
+     * @return create result code
+     */
+    public Map<String, Object> activateUser(User loginUser, String userName) {
+        Map<String, Object> result = new HashMap<>();
+        result.put(Constants.STATUS, false);
+
+        if (!isAdmin(loginUser)) {
+            putMsg(result, Status.USER_NO_OPERATION_PERM);
+            return result;
+        }
+
+        if (!CheckUtils.checkUserName(userName)){
+            putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, userName);
+            return result;
+        }
+
+        User user = userMapper.queryByUserNameAccurately(userName);
+
+        if (user == null) {
+            putMsg(result, Status.USER_NOT_EXIST, userName);
+            return result;
+        }
+
+        if (user.getState() != Flag.NO.ordinal()) {
+            putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, userName);
+            return result;
+        }
+
+        user.setState(Flag.YES.ordinal());
+        Date now = new Date();
+        user.setUpdateTime(now);
+        userMapper.updateById(user);
+        User responseUser = userMapper.queryByUserNameAccurately(userName);
+        putMsg(result, Status.SUCCESS);
+        result.put(Constants.DATA_LIST, responseUser);
+        return result;
+    }
 }

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkFlowLineageService.java

@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.api.service;
+
+import org.apache.dolphinscheduler.api.enums.Status;
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper;
+import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage;
+import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+
+@Service
+public class WorkFlowLineageService extends BaseService {
+
+    @Autowired
+    private WorkFlowLineageMapper workFlowLineageMapper;
+
+    public Map<String, Object> queryWorkFlowLineageByName(String workFlowName, int projectId) {
+        Map<String, Object> result = new HashMap<>(5);
+        List<WorkFlowLineage> workFlowLineageList = workFlowLineageMapper.queryByName(workFlowName, projectId);
+        result.put(Constants.DATA_LIST, workFlowLineageList);
+        putMsg(result, Status.SUCCESS);
+        return result;
+    }
+
+    private List<WorkFlowRelation> getWorkFlowRelationRecursion(Set<Integer> ids, List<WorkFlowRelation> workFlowRelations,Set<Integer> sourceIds) {
+        for(int id : ids) {
+            sourceIds.addAll(ids);
+            List<WorkFlowRelation> workFlowRelationsTmp = workFlowLineageMapper.querySourceTarget(id);
+            if(workFlowRelationsTmp != null && !workFlowRelationsTmp.isEmpty()) {
+                Set<Integer> idsTmp = new HashSet<>();
+                for(WorkFlowRelation workFlowRelation:workFlowRelationsTmp) {
+                    if(!sourceIds.contains(workFlowRelation.getTargetWorkFlowId())){
+                        idsTmp.add(workFlowRelation.getTargetWorkFlowId());
+                    }
+                }
+                workFlowRelations.addAll(workFlowRelationsTmp);
+                getWorkFlowRelationRecursion(idsTmp, workFlowRelations,sourceIds);
+            }
+        }
+        return workFlowRelations;
+    }
+
+    public Map<String, Object> queryWorkFlowLineageByIds(Set<Integer> ids,int projectId) {
+        Map<String, Object> result = new HashMap<>(5);
+        List<WorkFlowLineage> workFlowLineageList = workFlowLineageMapper.queryByIds(ids, projectId);
+        Map<String, Object> workFlowLists = new HashMap<>(5);
+        Set<Integer> idsV = new HashSet<>();
+        if(ids == null || ids.isEmpty()){
+            for(WorkFlowLineage workFlowLineage:workFlowLineageList) {
+                idsV.add(workFlowLineage.getWorkFlowId());
+            }
+        } else {
+            idsV = ids;
+        }
+        List<WorkFlowRelation> workFlowRelations = new ArrayList<>();
+        Set<Integer> sourceIds = new HashSet<>();
+        getWorkFlowRelationRecursion(idsV, workFlowRelations, sourceIds);
+
+        Set<Integer> idSet = new HashSet<>();
+        //If the incoming parameter is not empty, you need to add downstream workflow detail attributes
+        if(ids != null && !ids.isEmpty()) {
+            for(WorkFlowRelation workFlowRelation : workFlowRelations) {
+                idSet.add(workFlowRelation.getTargetWorkFlowId());
+            }
+            for(int id : ids){
+                idSet.remove(id);
+            }
+            if(!idSet.isEmpty()) {
+                workFlowLineageList.addAll(workFlowLineageMapper.queryByIds(idSet, projectId));
+            }
+        }
+
+        workFlowLists.put("workFlowList",workFlowLineageList);
+        workFlowLists.put("workFlowRelationList",workFlowRelations);
+        result.put(Constants.DATA_LIST, workFlowLists);
+        putMsg(result, Status.SUCCESS);
+        return result;
+    }
+}

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java

@@ -16,15 +16,12 @@
  */
 package org.apache.dolphinscheduler.api.service;
 
-import com.baomidou.mybatisplus.core.metadata.IPage;
-import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import org.apache.dolphinscheduler.api.enums.Status;
 import org.apache.dolphinscheduler.api.utils.PageInfo;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.utils.CollectionUtils;
 import org.apache.dolphinscheduler.common.utils.DateUtils;
 import org.apache.dolphinscheduler.common.utils.StringUtils;
-import org.apache.dolphinscheduler.dao.entity.AccessToken;
 import org.apache.dolphinscheduler.dao.entity.User;
 import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
 import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
@@ -149,8 +146,8 @@ public class WorkerGroupService extends BaseService {
                 if (isPaging){
                     wg.setIpList(childrenNodes);
                     String registeredIpValue = zookeeperCachedOperator.get(workerGroupPath + "/" + childrenNodes.get(0));
-                    wg.setCreateTime(DateUtils.stringToDate(registeredIpValue.split(",")[3]));
+                    wg.setCreateTime(DateUtils.stringToDate(registeredIpValue.split(",")[6]));
-                    wg.setUpdateTime(DateUtils.stringToDate(registeredIpValue.split(",")[4]));
+                    wg.setUpdateTime(DateUtils.stringToDate(registeredIpValue.split(",")[7]));
                 }
                 workerGroups.add(wg);
             }

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java

@@ -149,6 +149,7 @@ public class ZooKeeperState {
 			return sendThread.ret;
 		} catch (InterruptedException e) {
 			logger.error("send " + cmd + " to server " + host + ":" + port + " failed!", e);
+			Thread.currentThread().interrupt();
 		}
 		return "";
 	}

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitor.java

@@ -82,7 +82,6 @@ public class ZookeeperMonitor extends AbstractZKClient {
 					state.getZookeeperInfo();
 				}
 
-				String hostName = zookeeperServer;
 				int connections = state.getConnections();
 				int watches = state.getWatches();
 				long sent = state.getSent();
@@ -95,7 +94,7 @@ public class ZookeeperMonitor extends AbstractZKClient {
 				int status = ok ? 1 : 0;
 				Date date = new Date();
 
-				ZookeeperRecord zookeeperRecord = new ZookeeperRecord(hostName,connections,watches,sent,received,mode,minLatency,avgLatency,maxLatency,nodeCount,status,date);
+				ZookeeperRecord zookeeperRecord = new ZookeeperRecord(zookeeperServer,connections,watches,sent,received,mode,minLatency,avgLatency,maxLatency,nodeCount,status,date);
 				list.add(zookeeperRecord);
 
 			}

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java

@@ -74,7 +74,8 @@ public class ProcessInstanceControllerTest extends AbstractControllerTest {
                 .andReturn();
 
         Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
-        Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.getCode().intValue());
+        assert result != null;
+        Assert.assertEquals(Status.PROJECT_NOT_FOUNT.getCode(),result.getCode().intValue());
         logger.info(mvcResult.getResponse().getContentAsString());
     }
 

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java

@@ -285,6 +285,21 @@ public class UsersControllerTest extends AbstractControllerTest{
 
         Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
         Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
-        logger.info(mvcResult.getResponse().getContentAsString());
+    }
+
+    @Test
+    public void testActivateUser() throws Exception {
+        MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
+        paramsMap.add("userName","user_test");
+
+        MvcResult mvcResult = mockMvc.perform(post("/users/activate")
+                .header(SESSION_ID, sessionId)
+                .params(paramsMap))
+                .andExpect(status().isOk())
+                .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
+                .andReturn();
+
+        Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
+        Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
     }
 }

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageControllerTest.java

@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.api.controller;
+
+import org.apache.dolphinscheduler.api.enums.Status;
+import org.apache.dolphinscheduler.api.utils.Result;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.http.MediaType;
+import org.springframework.test.web.servlet.MvcResult;
+import org.springframework.util.LinkedMultiValueMap;
+import org.springframework.util.MultiValueMap;
+
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+public class WorkFlowLineageControllerTest extends AbstractControllerTest {
+    private static Logger logger = LoggerFactory.getLogger(WorkFlowLineageControllerTest.class);
+
+    @Test
+    public void testQueryWorkFlowLineageByName() throws Exception {
+        MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
+        paramsMap.add("searchVal","test");
+        MvcResult mvcResult = mockMvc.perform(get("/lineages/1/list-name")
+                .header("sessionId", sessionId)
+                .params(paramsMap))
+                .andExpect(status().isOk())
+                .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
+                .andReturn();
+        Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
+        Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
+        logger.info(mvcResult.getResponse().getContentAsString());
+
+    }
+
+    @Test
+    public  void testQueryWorkFlowLineageByIds() throws Exception {
+        MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
+        paramsMap.add("ids","1");
+        MvcResult mvcResult = mockMvc.perform(get("/lineages/1/list-ids")
+                .header("sessionId", sessionId)
+                .params(paramsMap))
+                .andExpect(status().isOk())
+                .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
+                .andReturn();
+        Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
+        Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
+        logger.info(mvcResult.getResponse().getContentAsString());
+    }
+
+}

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java

@@ -180,7 +180,7 @@ public class DataAnalysisServiceTest {
 
         List<ExecuteStatusCount> taskInstanceStateCounts = new ArrayList<>(1);
         ExecuteStatusCount executeStatusCount = new ExecuteStatusCount();
-        executeStatusCount.setExecutionStatus(ExecutionStatus.RUNNING_EXEUTION);
+        executeStatusCount.setExecutionStatus(ExecutionStatus.RUNNING_EXECUTION);
         taskInstanceStateCounts.add(executeStatusCount);
 
         return  taskInstanceStateCounts;

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java

@@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.DbConnectType;
 import org.apache.dolphinscheduler.common.enums.DbType;
 import org.apache.dolphinscheduler.common.enums.UserType;
+import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.apache.dolphinscheduler.dao.entity.DataSource;
 import org.apache.dolphinscheduler.dao.entity.User;
 import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
@@ -103,7 +104,28 @@ public class DataSourceServiceTest {
     public void buildParameter(){
         String param = dataSourceService.buildParameter("","", DbType.ORACLE, "192.168.9.1","1521","im"
                 ,"","test","test", DbConnectType.ORACLE_SERVICE_NAME,"");
-        String expected = "{\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}";
+        String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}";
         Assert.assertEquals(expected, param);
     }
+
+    @Test
+    public void buildParameterWithDecodePassword(){
+        PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
+        String param = dataSourceService.buildParameter("name","desc", DbType.MYSQL, "192.168.9.1","1521","im"
+                ,"","test","123456", null,"");
+        String expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}";
+        Assert.assertEquals(expected, param);
+
+
+        PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
+        param = dataSourceService.buildParameter("name","desc", DbType.MYSQL, "192.168.9.1","1521","im"
+                ,"","test","123456", null,"");
+        expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"123456\"}";
+        Assert.assertEquals(expected, param);
+    }
+
+
+
+
+
 }

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java

@@ -474,7 +474,7 @@ public class ProcessDefinitionServiceTest {
         ProcessInstance processInstance = new ProcessInstance();
         processInstance.setId(1);
         processInstance.setName("test_instance");
-        processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
+        processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
         processInstance.setHost("192.168.xx.xx");
         processInstance.setStartTime(new Date());
         processInstance.setEndTime(new Date());
@@ -486,7 +486,7 @@ public class ProcessDefinitionServiceTest {
         taskInstance.setTaskType("SHELL");
         taskInstance.setId(1);
         taskInstance.setName("test_task_instance");
-        taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
+        taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
         taskInstance.setHost("192.168.xx.xx");
 
         //task instance not exist

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java

@@ -147,6 +147,40 @@ public class ProcessInstanceServiceTest {
 
     }
 
+    @Test
+    public void testQueryTopNLongestRunningProcessInstance() {
+        String projectName = "project_test1";
+        User loginUser = getAdminUser();
+        Map<String, Object> result = new HashMap<>(5);
+        putMsg(result, Status.PROJECT_NOT_FOUNT, projectName);
+        int size=10;
+        String startTime="2020-01-01 00:00:00";
+        String endTime="2020-08-02 00:00:00";
+        Date start = DateUtils.getScheduleDate(startTime);
+        Date end = DateUtils.getScheduleDate(endTime);
+
+        //project auth fail
+        when(projectMapper.queryByName(projectName)).thenReturn(null);
+        when(projectService.checkProjectAndAuth(loginUser, null, projectName)).thenReturn(result);
+        Map<String, Object> proejctAuthFailRes = processInstanceService.queryTopNLongestRunningProcessInstance(loginUser,projectName,size,startTime,endTime);
+        Assert.assertEquals(Status.PROJECT_NOT_FOUNT, proejctAuthFailRes.get(Constants.STATUS));
+
+        //project auth success
+        putMsg(result, Status.SUCCESS, projectName);
+        Project project = getProject(projectName);
+        ProcessInstance processInstance = getProcessInstance();
+        List<ProcessInstance> processInstanceList = new ArrayList<>();
+        processInstanceList.add(processInstance);
+        when(projectMapper.queryByName(projectName)).thenReturn(project);
+        when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result);
+        when(usersService.queryUser(loginUser.getId())).thenReturn(loginUser);
+        when(usersService.getUserIdByName(loginUser.getUserName())).thenReturn(loginUser.getId());
+        when(usersService.queryUser(processInstance.getExecutorId())).thenReturn(loginUser);
+        Map<String, Object> successRes = processInstanceService.queryTopNLongestRunningProcessInstance(loginUser,projectName,size,startTime,endTime);
+
+        Assert.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS));
+    }
+
     @Test
     public void testQueryProcessInstanceById() {
         String projectName = "project_test1";
@@ -307,7 +341,7 @@ public class ProcessInstanceServiceTest {
 
         //process instance not finish
         when(processService.findProcessInstanceDetailById(1)).thenReturn(processInstance);
-        processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
+        processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
         Map<String, Object> processInstanceNotFinishRes = processInstanceService.updateProcessInstance(loginUser, projectName, 1,
                 shellJson, "2020-02-21 00:00:00", true, Flag.YES, "", "");
         Assert.assertEquals(Status.PROCESS_INSTANCE_STATE_OPERATION_ERROR, processInstanceNotFinishRes.get(Constants.STATUS));
@@ -414,7 +448,7 @@ public class ProcessInstanceServiceTest {
         ProcessInstance processInstance = getProcessInstance();
         processInstance.setProcessInstanceJson(shellJson);
         TaskInstance taskInstance = getTaskInstance();
-        taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
+        taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
         taskInstance.setStartTime(new Date());
         when(processInstanceMapper.queryDetailById(1)).thenReturn(processInstance);
         when(taskInstanceMapper.queryByInstanceIdAndName(Mockito.anyInt(), Mockito.any())).thenReturn(taskInstance);

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java

@@ -462,42 +462,87 @@ public class UsersServiceTest {
         try {
             //userName error
             Map<String, Object> result = usersService.registerUser(userName, userPassword, repeatPassword, email);
-            logger.info(result.toString());
             Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
 
             userName = "userTest0002";
             userPassword = "userTest000111111111111111";
             //password error
             result = usersService.registerUser(userName, userPassword, repeatPassword, email);
-            logger.info(result.toString());
             Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
 
             userPassword = "userTest0002";
             email = "1q.com";
             //email error
             result = usersService.registerUser(userName, userPassword, repeatPassword, email);
-            logger.info(result.toString());
             Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
 
             //repeatPassword error
             email = "7400@qq.com";
             repeatPassword = "userPassword";
             result = usersService.registerUser(userName, userPassword, repeatPassword, email);
-            logger.info(result.toString());
             Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
 
             //success
             repeatPassword = "userTest0002";
             result = usersService.registerUser(userName, userPassword, repeatPassword, email);
-            logger.info(result.toString());
             Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
 
         } catch (Exception e) {
-            logger.error(Status.CREATE_USER_ERROR.getMsg(),e);
             Assert.assertTrue(false);
         }
     }
 
+
+    @Test
+    public void testActivateUser() {
+        User user = new User();
+        user.setUserType(UserType.GENERAL_USER);
+        String userName = "userTest0002~";
+        try {
+            //not admin
+            Map<String, Object> result = usersService.activateUser(user, userName);
+            Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
+
+            //userName error
+            user.setUserType(UserType.ADMIN_USER);
+            result = usersService.activateUser(user, userName);
+            Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
+
+            //user not exist
+            userName = "userTest10013";
+            result = usersService.activateUser(user, userName);
+            Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS));
+
+            //user state error
+            userName = "userTest0001";
+            when(userMapper.queryByUserNameAccurately(userName)).thenReturn(getUser());
+            result = usersService.activateUser(user, userName);
+            Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
+
+            //success
+            when(userMapper.queryByUserNameAccurately(userName)).thenReturn(getDisabledUser());
+            result = usersService.activateUser(user, userName);
+            Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
+        } catch (Exception e) {
+            Assert.assertTrue(false);
+        }
+    }
+
+    /**
+     * get disabled user
+     * @return
+     */
+    private User getDisabledUser() {
+
+        User user = new User();
+        user.setUserType(UserType.GENERAL_USER);
+        user.setUserName("userTest0001");
+        user.setUserPassword("userTest0001");
+        user.setState(0);
+        return user;
+    }
+
+
     /**
      * get user
      * @return

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkFlowLineageServiceTest.java

@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.api.service;
+
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.utils.EncryptionUtils;
+import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage;
+import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation;
+import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.*;
+
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class WorkFlowLineageServiceTest {
+
+    @InjectMocks
+    private WorkFlowLineageService workFlowLineageService;
+
+    @Mock
+    private WorkFlowLineageMapper workFlowLineageMapper;
+
+    @Test
+    public void testQueryWorkFlowLineageByName() {
+        String searchVal = "test";
+        when(workFlowLineageMapper.queryByName(searchVal, 1)).thenReturn(getWorkFlowLineages());
+        Map<String, Object> result = workFlowLineageService.queryWorkFlowLineageByName(searchVal,1);
+        List<WorkFlowLineage> workFlowLineageList = (List<WorkFlowLineage>)result.get(Constants.DATA_LIST);
+        Assert.assertTrue(workFlowLineageList.size()>0);
+    }
+
+    @Test
+    public void testQueryWorkFlowLineageByIds() {
+
+        Set<Integer> ids = new HashSet<>();
+        ids.add(1);
+        ids.add(2);
+
+        when(workFlowLineageMapper.queryByIds(ids, 1)).thenReturn(getWorkFlowLineages());
+        when(workFlowLineageMapper.querySourceTarget(1)).thenReturn(getWorkFlowRelation());
+        Map<String, Object> result = workFlowLineageService.queryWorkFlowLineageByIds(ids,1);
+        Map<String, Object> workFlowLists = (Map<String, Object>)result.get(Constants.DATA_LIST);
+        List<WorkFlowLineage> workFlowLineages = (List<WorkFlowLineage>)workFlowLists.get("workFlowList");
+        List<WorkFlowRelation> workFlowRelations = (List<WorkFlowRelation>)workFlowLists.get("workFlowRelationList");
+        Assert.assertTrue(workFlowLineages.size()>0);
+        Assert.assertTrue(workFlowRelations.size()>0);
+    }
+
+    private List<WorkFlowLineage> getWorkFlowLineages() {
+        List<WorkFlowLineage> workFlowLineages = new ArrayList<>();
+        WorkFlowLineage workFlowLineage = new WorkFlowLineage();
+        workFlowLineage.setWorkFlowId(1);
+        workFlowLineage.setWorkFlowName("testdag");
+        workFlowLineages.add(workFlowLineage);
+        return workFlowLineages;
+    }
+
+    private List<WorkFlowRelation> getWorkFlowRelation(){
+        List<WorkFlowRelation> workFlowRelations = new ArrayList<>();
+        WorkFlowRelation workFlowRelation = new WorkFlowRelation();
+        workFlowRelation.setSourceWorkFlowId(1);
+        workFlowRelation.setTargetWorkFlowId(2);
+        workFlowRelations.add(workFlowRelation);
+        return workFlowRelations;
+    }
+
+}

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java

@@ -79,7 +79,7 @@ public class WorkerGroupServiceTest {
 
         Mockito.when(zookeeperCachedOperator.getChildrenKeys(workerPath + "/default")).thenReturn(defaultIpList);
 
-        Mockito.when(zookeeperCachedOperator.get(workerPath + "/default" + "/" + defaultIpList.get(0))).thenReturn("0.02,0.23,0.03,2020-05-08 11:24:14,2020-05-08 14:22:24");
+        Mockito.when(zookeeperCachedOperator.get(workerPath + "/default" + "/" + defaultIpList.get(0))).thenReturn("0.01,0.17,0.03,25.83,8.0,1.0,2020-07-21 11:17:59,2020-07-21 14:39:20,0,13238");
     }
 
     /**

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@@ -831,7 +831,7 @@ public final class Constants {
 
     public static final int[] NOT_TERMINATED_STATES = new int[]{
             ExecutionStatus.SUBMITTED_SUCCESS.ordinal(),
-            ExecutionStatus.RUNNING_EXEUTION.ordinal(),
+            ExecutionStatus.RUNNING_EXECUTION.ordinal(),
             ExecutionStatus.READY_PAUSE.ordinal(),
             ExecutionStatus.READY_STOP.ordinal(),
             ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal(),
@@ -978,10 +978,23 @@ public final class Constants {
     public static final int NORAML_NODE_STATUS = 0;
     public static final int ABNORMAL_NODE_STATUS = 1;
 
+    public static final String START_TIME = "start time";
+    public static final String END_TIME = "end time";
+    /**
+     * system line separator
+     */
+    public static final String SYSTEM_LINE_SEPARATOR = System.getProperty("line.separator");
+
     /**
      * net system properties
      */
     public static final String DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE = "dolphin.scheduler.network.interface.preferred";
 
 
+    /**
+     * datasource encryption salt
+     */
+    public static final String DATASOURCE_ENCRYPTION_SALT_DEFAULT = "!@#$%^&*";
+    public static final String DATASOURCE_ENCRYPTION_ENABLE = "datasource.encryption.enable";
+    public static final String DATASOURCE_ENCRYPTION_SALT = "datasource.encryption.salt";
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java

@@ -43,7 +43,7 @@ public enum ExecutionStatus {
      * 11 waiting depend node complete
      */
     SUBMITTED_SUCCESS(0, "submit success"),
-    RUNNING_EXEUTION(1, "running"),
+    RUNNING_EXECUTION(1, "running"),
     READY_PAUSE(2, "ready pause"),
     PAUSE(3, "pause"),
     READY_STOP(4, "ready stop"),
@@ -126,7 +126,7 @@ public enum ExecutionStatus {
      * @return status
      */
    public boolean typeIsRunning(){
-       return this == RUNNING_EXEUTION || this == WAITTING_DEPEND;
+       return this == RUNNING_EXECUTION || this == WAITTING_DEPEND;
    }
 
     /**

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskStateType.java

@@ -50,7 +50,7 @@ public enum TaskStateType {
                 };
             case RUNNING:
                 return new int[]{ExecutionStatus.SUBMITTED_SUCCESS.ordinal(),
-                        ExecutionStatus.RUNNING_EXEUTION.ordinal(),
+                        ExecutionStatus.RUNNING_EXECUTION.ordinal(),
                         ExecutionStatus.READY_PAUSE.ordinal(),
                         ExecutionStatus.READY_STOP.ordinal()};
             case WAITTING:

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java

@@ -36,6 +36,7 @@ public enum TaskType {
      * 10 DATAX
      * 11 CONDITIONS
      * 12 SQOOP
+     * 13 WATERDROP
      */
     SHELL(0, "shell"),
     SQL(1, "sql"),
@@ -49,7 +50,8 @@ public enum TaskType {
     HTTP(9, "http"),
     DATAX(10, "datax"),
     CONDITIONS(11, "conditions"),
-    SQOOP(12, "sqoop");
+    SQOOP(12, "sqoop"),
+    WATERDROP(13, "waterdrop");
 
     TaskType(int code, String descp){
         this.code = code;

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java

@@ -78,7 +78,7 @@ public abstract class AbstractShell {
   /**
    * If or not script finished executing
    */
-  private volatile AtomicBoolean completed;
+  private AtomicBoolean completed;
   
   public AbstractShell() {
     this(0L);
@@ -202,7 +202,7 @@ public abstract class AbstractShell {
       } catch (InterruptedException ie) {
         logger.warn("Interrupted while reading the error and in stream", ie);
       }
-      completed.set(true);
+      completed.compareAndSet(false,true);
       //the timeout thread handling
       //taken care in finally block
       if (exitCode != 0 || errMsg.length() > 0) {

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java

@@ -231,6 +231,8 @@ public class ThreadUtils {
     public static void sleep(final long millis) {
         try {
             Thread.sleep(millis);
-        } catch (final InterruptedException ignore) {}
+        } catch (final InterruptedException ignore) {
+            Thread.currentThread().interrupt();
+        }
     }
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java

@@ -16,6 +16,7 @@
  */
 package org.apache.dolphinscheduler.common.utils;
 
+import org.apache.commons.codec.binary.Base64;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.ResUploadType;
 import org.apache.hadoop.conf.Configuration;
@@ -23,8 +24,8 @@ import org.apache.hadoop.security.UserGroupInformation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
 import java.net.URL;
+import java.nio.charset.StandardCharsets;
 
 /**
  * common utils
@@ -32,6 +33,8 @@ import java.net.URL;
 public class CommonUtils {
   private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class);
 
+  private static final Base64 BASE64 = new Base64();
+
   private CommonUtils() {
     throw new IllegalStateException("CommonUtils class");
   }
@@ -90,4 +93,45 @@ public class CommonUtils {
               PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH));
     }
   }
+
+  /**
+   * encode password
+   * @param password
+   * @return
+   */
+  public static String encodePassword(String password) {
+    if(StringUtils.isEmpty(password)){return StringUtils.EMPTY; }
+    //if encryption is not turned on, return directly
+    boolean encryptionEnable =  PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false);
+    if ( !encryptionEnable){ return password; }
+
+    // Using Base64 + salt to process password
+    String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
+    String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8)))   ;
+    return  new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8)));
+  }
+
+  /**
+   * decode password
+   * @param password
+   * @return
+   */
+  public static String decodePassword(String password) {
+    if(StringUtils.isEmpty(password)){return StringUtils.EMPTY ; }
+
+    //if encryption is not turned on, return directly
+    boolean encryptionEnable =  PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false);
+    if ( !encryptionEnable){ return password; }
+
+    // Using Base64 + salt to process password
+    String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
+    String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8) ;
+    if(!passwordWithSalt.startsWith(salt)){
+      logger.warn("There is a password and salt mismatch: {} ",password);
+      return password;
+    }
+    return new String(BASE64.decode(passwordWithSalt.substring(salt.length())), StandardCharsets.UTF_8) ;
+  }
+
+
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java

@@ -448,7 +448,7 @@ public class HadoopUtils implements Closeable {
 
             case Constants.RUNNING:
             default:
-                return ExecutionStatus.RUNNING_EXEUTION;
+                return ExecutionStatus.RUNNING_EXECUTION;
         }
     }
 

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java

@@ -18,17 +18,33 @@ package org.apache.dolphinscheduler.common.utils;
 
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.http.HttpEntity;
+import org.apache.http.client.config.AuthSchemes;
+import org.apache.http.client.config.CookieSpecs;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
 import java.io.IOException;
-
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.X509Certificate;
+import java.util.Arrays;
 /**
  * http utils
  */
@@ -37,22 +53,81 @@ public class HttpUtils {
 
 	public static final Logger logger = LoggerFactory.getLogger(HttpUtils.class);
 
+	private HttpUtils() {
+
+	}
+
+	public static CloseableHttpClient getInstance(){
+		return HttpClientInstance.httpClient;
+	}
+
+	private static class HttpClientInstance{
+		private static final CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(requestConfig).build();
+	}
+
+
+	private static PoolingHttpClientConnectionManager cm;
+
+	private static SSLContext ctx = null;
+
+	private static SSLConnectionSocketFactory socketFactory;
+
+	private static RequestConfig requestConfig;
+
+	private static Registry<ConnectionSocketFactory> socketFactoryRegistry;
+
+	private static X509TrustManager xtm = new X509TrustManager() {
+		@Override
+		public void checkClientTrusted(X509Certificate[] chain, String authType) {
+		}
+
+		@Override
+		public void checkServerTrusted(X509Certificate[] chain, String authType) {
+		}
+
+		@Override
+		public X509Certificate[] getAcceptedIssuers() {
+			return null;
+		}
+	};
+
+	static {
+		try {
+			ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS);
+			ctx.init(null, new TrustManager[] { xtm }, null);
+		} catch (NoSuchAlgorithmException e) {
+			logger.error("SSLContext init with NoSuchAlgorithmException", e);
+		} catch (KeyManagementException e) {
+			logger.error("SSLContext init with KeyManagementException", e);
+		}
+		socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE);
+		/** set timeout、request time、socket timeout */
+		requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES)
+				.setExpectContinueEnabled(Boolean.TRUE)
+				.setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST))
+				.setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC))
+				.setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT)
+				.setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true)
+				.build();
+		socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
+				.register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build();
+		cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+		cm.setDefaultMaxPerRoute(60);
+		cm.setMaxTotal(100);
+
+	}
+
+
+
 	/**
 	 * get http request content
 	 * @param url url
 	 * @return http get request response content
 	 */
 	public static String get(String url){
-		CloseableHttpClient httpclient = HttpClients.createDefault();
+		CloseableHttpClient httpclient = HttpUtils.getInstance();
 
 		HttpGet httpget = new HttpGet(url);
-		/** set timeout、request time、socket timeout */
-		RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT)
-				.setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT)
-				.setSocketTimeout(Constants.SOCKET_TIMEOUT)
-				.setRedirectsEnabled(true)
-				.build();
-		httpget.setConfig(requestConfig);
 		String responseContent = null;
 		CloseableHttpResponse response = null;
 
@@ -85,12 +160,6 @@ public class HttpUtils {
 				httpget.releaseConnection();
 				httpget.abort();
 			}
-
-			try {
-				httpclient.close();
-			} catch (IOException e) {
-				logger.error(e.getMessage(),e);
-			}
 		}
 		return responseContent;
 	}

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java

@@ -207,7 +207,7 @@ public class ParameterUtils {
   public static String handleEscapes(String inputString){
 
     if(StringUtils.isNotEmpty(inputString)){
-      return inputString.replace("%", "////%");
+      return inputString.replace("%", "////%").replaceAll("[\n|\r\t]", "_");
     }
     return inputString;
   }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java

@@ -250,4 +250,14 @@ public class PropertyUtils {
         }
         return matchedProperties;
     }
+
+    /**
+     *
+     * @param key
+     * @param value
+     */
+    public static void setValue(String key, String value) {
+        properties.setProperty(key,value);
+    }
+
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java

@@ -53,6 +53,8 @@ public class TaskParametersUtils {
       switch (EnumUtils.getEnum(TaskType.class,taskType)) {
         case SUB_PROCESS:
           return JSONUtils.parseObject(parameter, SubProcessParameters.class);
+        case WATERDROP:
+          return JSONUtils.parseObject(parameter, ShellParameters.class);
         case SHELL:
           return JSONUtils.parseObject(parameter, ShellParameters.class);
         case PROCEDURE:

dolphinscheduler-common/src/main/resources/common.properties

@@ -68,3 +68,7 @@ development.state=false
 
 # kerberos tgt expire time, unit is hours
 kerberos.expire.time=2
+
+# datasource encryption salt
+datasource.encryption.enable=false
+datasource.encryption.salt=!@#$%^&*

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java

@@ -16,6 +16,7 @@
  */
 package org.apache.dolphinscheduler.common.utils;
 
+import org.apache.dolphinscheduler.common.Constants;
 import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -89,4 +90,42 @@ public class CommonUtilsTest {
         }
         Assert.assertTrue(true);
     }
+
+    @Test
+    public void encodePassword() {
+
+        PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
+
+        Assert.assertEquals("",CommonUtils.encodePassword(""));
+        Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==",CommonUtils.encodePassword("123456"));
+        Assert.assertEquals("IUAjJCVeJipJVkZCV2xoVFYwQT0=",CommonUtils.encodePassword("!QAZXSW@"));
+        Assert.assertEquals("IUAjJCVeJipOV1JtWjJWeUtFQT0=",CommonUtils.encodePassword("5dfger(@"));
+
+        PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
+
+        Assert.assertEquals("",CommonUtils.encodePassword(""));
+        Assert.assertEquals("123456",CommonUtils.encodePassword("123456"));
+        Assert.assertEquals("!QAZXSW@",CommonUtils.encodePassword("!QAZXSW@"));
+        Assert.assertEquals("5dfger(@",CommonUtils.encodePassword("5dfger(@"));
+
+    }
+
+    @Test
+    public void decodePassword() {
+
+        PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true");
+
+        Assert.assertEquals("", CommonUtils.decodePassword(""));
+        Assert.assertEquals("123456", CommonUtils.decodePassword("IUAjJCVeJipNVEl6TkRVMg=="));
+        Assert.assertEquals("!QAZXSW@", CommonUtils.decodePassword("IUAjJCVeJipJVkZCV2xoVFYwQT0="));
+        Assert.assertEquals("5dfger(@", CommonUtils.decodePassword("IUAjJCVeJipOV1JtWjJWeUtFQT0="));
+
+        PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false");
+
+        Assert.assertEquals("", CommonUtils.decodePassword(""));
+        Assert.assertEquals("123456", CommonUtils.decodePassword("123456"));
+        Assert.assertEquals("!QAZXSW@", CommonUtils.decodePassword("!QAZXSW@"));
+        Assert.assertEquals("5dfger(@", CommonUtils.decodePassword("5dfger(@"));
+    }
+
 }

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java

@@ -16,6 +16,7 @@
  */
 package org.apache.dolphinscheduler.common.utils;
 
+import org.apache.dolphinscheduler.common.Constants;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -67,4 +68,17 @@ public class FileUtilsTest {
             Assert.assertTrue(false);
         }
     }
+
+    @Test
+    public void testSetValue() {
+        try {
+            PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
+            Assert.assertTrue(PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE));
+            PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
+            Assert.assertFalse(PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE));
+        } catch (Exception e) {
+            Assert.assertTrue(false);
+        }
+    }
+
 }

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java

@@ -17,6 +17,7 @@
 package org.apache.dolphinscheduler.common.utils;
 
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.http.impl.client.CloseableHttpClient;
 import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -42,4 +43,11 @@ public class HttpUtilsTest {
 		result = HttpUtils.get("https://123.333.111.33/ccc");
 		Assert.assertNull(result);
 	}
+
+	@Test
+	public void testGetHttpClient() {
+		CloseableHttpClient httpClient1 = HttpUtils.getInstance();
+		CloseableHttpClient httpClient2 = HttpUtils.getInstance();
+		Assert.assertEquals(httpClient1, httpClient2);
+	}
 }

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java

@@ -17,6 +17,7 @@
 package org.apache.dolphinscheduler.common.utils;
 
 import org.apache.dolphinscheduler.common.Constants;
+import org.junit.Assert;
 import org.junit.Test;
 
 import static org.junit.Assert.assertNotNull;
@@ -26,6 +27,5 @@ public class PropertyUtilsTest {
     @Test
     public void getString() {
         assertNotNull(PropertyUtils.getString(Constants.FS_DEFAULTFS));
-        assertNotNull(PropertyUtils.getInt("spring.redis.port"));
     }
 }

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java

@@ -20,6 +20,7 @@ import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
 import org.apache.dolphinscheduler.common.enums.DbType;
+import org.apache.dolphinscheduler.common.utils.CommonUtils;
 import org.apache.dolphinscheduler.common.utils.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -182,8 +183,12 @@ public abstract class BaseDataSource {
     this.user = user;
   }
 
+  /**
+   * password need decode
+   * @return
+   */
   public String getPassword() {
-    return password;
+    return CommonUtils.decodePassword(password);
   }
 
   public void setPassword(String password) {

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java

@@ -82,6 +82,8 @@ public class MySQLDataSource extends BaseDataSource {
 
   @Override
   public String getPassword() {
+    // password need decode
+    password = super.getPassword();
     if(password.contains(sensitiveParam)){
       logger.warn("sensitive param : {} in password field is filtered", sensitiveParam);
       password = password.replace(sensitiveParam, "");

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java

@@ -26,6 +26,8 @@ import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.configuration.PropertiesConfiguration;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.dao.utils.PropertyUtils;
+import org.apache.ibatis.mapping.DatabaseIdProvider;
+import org.apache.ibatis.mapping.VendorDatabaseIdProvider;
 import org.apache.ibatis.session.SqlSession;
 import org.apache.ibatis.session.SqlSessionFactory;
 import org.apache.ibatis.type.JdbcType;
@@ -39,6 +41,8 @@ import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
 import org.springframework.core.io.support.ResourcePatternResolver;
 import org.springframework.jdbc.datasource.DataSourceTransactionManager;
 
+import java.util.Properties;
+
 
 /**
  * data source connection factory
@@ -129,6 +133,7 @@ public class SpringConnectionFactory {
         ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
         sqlSessionFactoryBean.setMapperLocations(resolver.getResources("org/apache/dolphinscheduler/dao/mapper/*Mapper.xml"));
         sqlSessionFactoryBean.setTypeEnumsPackage("org.apache.dolphinscheduler.*.enums");
+        sqlSessionFactoryBean.setDatabaseIdProvider(databaseIdProvider());
         return sqlSessionFactoryBean.getObject();
     }
 
@@ -142,4 +147,13 @@ public class SpringConnectionFactory {
         return new SqlSessionTemplate(sqlSessionFactory());
     }
 
+    @Bean
+    public DatabaseIdProvider databaseIdProvider(){
+        DatabaseIdProvider databaseIdProvider = new VendorDatabaseIdProvider();
+        Properties properties = new Properties();
+        properties.setProperty("MySQL", "mysql");
+        properties.setProperty("PostgreSQL", "pg");
+        databaseIdProvider.setProperties(properties);
+        return databaseIdProvider;
+    }
 }

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java

@@ -14,14 +14,19 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.dao.entity;
 
-import com.fasterxml.jackson.annotation.JsonFormat;
 import org.apache.dolphinscheduler.common.enums.UdfType;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.StringUtils;
 import com.baomidou.mybatisplus.annotation.IdType;
 import com.baomidou.mybatisplus.annotation.TableId;
 import com.baomidou.mybatisplus.annotation.TableName;
-
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.KeyDeserializer;
+import java.io.IOException;
 import java.util.Date;
 
 /**
@@ -215,19 +220,17 @@ public class UdfFunc {
 
     @Override
     public String toString() {
-        return "UdfFunc{" +
+        return JSONUtils.toJsonString(this);
-                "id=" + id +
+    }
-                ", userId=" + userId +
+
-                ", funcName='" + funcName + '\'' +
+    public static  class UdfFuncDeserializer extends KeyDeserializer {
-                ", className='" + className + '\'' +
+
-                ", argTypes='" + argTypes + '\'' +
+        @Override
-                ", database='" + database + '\'' +
+        public Object deserializeKey(String key, DeserializationContext ctxt) throws IOException {
-                ", description='" + description + '\'' +
+            if (StringUtils.isBlank(key)) {
-                ", resourceId=" + resourceId +
+                return null;
-                ", resourceName='" + resourceName + '\'' +
+            }
-                ", type=" + type +
+            return JSONUtils.parseObject(key);
-                ", createTime=" + createTime +
+        }
-                ", updateTime=" + updateTime +
-                '}';
     }
 }

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowLineage.java

@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.dao.entity;
+
+import java.util.Date;
+
+public class WorkFlowLineage {
+    private int workFlowId;
+    private String workFlowName;
+    private String workFlowPublishStatus;
+    private Date scheduleStartTime;
+    private Date scheduleEndTime;
+    private String crontab;
+    private int schedulePublishStatus;
+    private String sourceWorkFlowId;
+
+    public String getSourceWorkFlowId() {
+        return sourceWorkFlowId;
+    }
+
+    public void setSourceWorkFlowId(String sourceWorkFlowId) {
+        this.sourceWorkFlowId = sourceWorkFlowId;
+    }
+
+    public int getWorkFlowId() {
+        return workFlowId;
+    }
+
+    public void setWorkFlowId(int workFlowId) {
+        this.workFlowId = workFlowId;
+    }
+
+    public String getWorkFlowName() {
+        return workFlowName;
+    }
+
+    public void setWorkFlowName(String workFlowName) {
+        this.workFlowName = workFlowName;
+    }
+
+    public String getWorkFlowPublishStatus() {
+        return workFlowPublishStatus;
+    }
+
+    public void setWorkFlowPublishStatus(String workFlowPublishStatus) {
+        this.workFlowPublishStatus = workFlowPublishStatus;
+    }
+
+    public Date getScheduleStartTime() {
+        return scheduleStartTime;
+    }
+
+    public void setScheduleStartTime(Date scheduleStartTime) {
+        this.scheduleStartTime = scheduleStartTime;
+    }
+
+    public Date getScheduleEndTime() {
+        return scheduleEndTime;
+    }
+
+    public void setScheduleEndTime(Date scheduleEndTime) {
+        this.scheduleEndTime = scheduleEndTime;
+    }
+
+    public String getCrontab() {
+        return crontab;
+    }
+
+    public void setCrontab(String crontab) {
+        this.crontab = crontab;
+    }
+
+    public int getSchedulePublishStatus() {
+        return schedulePublishStatus;
+    }
+
+    public void setSchedulePublishStatus(int schedulePublishStatus) {
+        this.schedulePublishStatus = schedulePublishStatus;
+    }
+}

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowRelation.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.dao.entity;
+
+public class WorkFlowRelation {
+    private int sourceWorkFlowId;
+    private int targetWorkFlowId;
+
+    public int getSourceWorkFlowId() {
+        return sourceWorkFlowId;
+    }
+
+    public void setSourceWorkFlowId(int sourceWorkFlowId) {
+        this.sourceWorkFlowId = sourceWorkFlowId;
+    }
+
+    public int getTargetWorkFlowId() {
+        return targetWorkFlowId;
+    }
+
+    public void setTargetWorkFlowId(int targetWorkFlowId) {
+        this.targetWorkFlowId = targetWorkFlowId;
+    }
+}

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java

@@ -193,4 +193,17 @@ public interface ProcessInstanceMapper extends BaseMapper<ProcessInstance> {
     ProcessInstance queryLastManualProcess(@Param("processDefinitionId") int definitionId,
                                            @Param("startTime") Date startTime,
                                            @Param("endTime") Date endTime);
+    /**
+     * query top n process instance order by running duration
+     * @param size
+     * @param status process instance status
+     * @param startTime
+     * @param endTime
+     * @return ProcessInstance list
+     */
+    List<ProcessInstance> queryTopNProcessInstance(@Param("size") int size,
+                                                   @Param("startTime") Date startTime,
+                                                   @Param("endTime") Date endTime,
+                                                   @Param("status")ExecutionStatus status);
+
 }

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapper.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.dao.mapper;
+
+import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage;
+import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation;
+import org.apache.ibatis.annotations.Param;
+import java.util.List;
+import java.util.Set;
+
+public interface WorkFlowLineageMapper {
+
+    public List<WorkFlowLineage> queryByName(@Param("searchVal") String searchVal, @Param("projectId") int projectId);
+
+    public List<WorkFlowLineage> queryByIds(@Param("ids") Set<Integer> ids, @Param("projectId") int projectId);
+
+    public List<WorkFlowRelation> querySourceTarget(@Param("id") int id);
+}

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java

@@ -98,14 +98,16 @@ public class DagHelper {
                 List<TaskNode> childNodeList = new ArrayList<>();
                 if (startNode == null) {
                     logger.error("start node name [{}] is not in task node list [{}] ",
-                        startNodeName,
+                            startNodeName,
-                        taskNodeList
+                            taskNodeList
                     );
                     continue;
                 } else if (TaskDependType.TASK_POST == taskDependType) {
-                    childNodeList = getFlowNodeListPost(startNode, taskNodeList);
+                    List<String> visitedNodeNameList = new ArrayList<>();
+                    childNodeList = getFlowNodeListPost(startNode, taskNodeList, visitedNodeNameList);
                 } else if (TaskDependType.TASK_PRE == taskDependType) {
-                    childNodeList = getFlowNodeListPre(startNode, recoveryNodeNameList, taskNodeList);
+                    List<String> visitedNodeNameList = new ArrayList<>();
+                    childNodeList = getFlowNodeListPre(startNode, recoveryNodeNameList, taskNodeList, visitedNodeNameList);
                 } else {
                     childNodeList.add(startNode);
                 }
@@ -128,14 +130,19 @@ public class DagHelper {
      * @param taskNodeList taskNodeList
      * @return task node list
      */
-    private static List<TaskNode> getFlowNodeListPost(TaskNode startNode, List<TaskNode> taskNodeList) {
+    private static List<TaskNode> getFlowNodeListPost(TaskNode startNode, List<TaskNode> taskNodeList, List<String> visitedNodeNameList) {
         List<TaskNode> resultList = new ArrayList<>();
         for (TaskNode taskNode : taskNodeList) {
             List<String> depList = taskNode.getDepList();
-            if (null != depList && null != startNode && depList.contains(startNode.getName())) {
+            if (null != depList && null != startNode && depList.contains(startNode.getName()) && !visitedNodeNameList.contains(taskNode.getName())) {
-                resultList.addAll(getFlowNodeListPost(taskNode, taskNodeList));
+                resultList.addAll(getFlowNodeListPost(taskNode, taskNodeList, visitedNodeNameList));
             }
         }
+        // why add (startNode != null) condition? for SonarCloud Quality Gate passed
+        if (null != startNode) {
+            visitedNodeNameList.add(startNode.getName());
+        }
+
         resultList.add(startNode);
         return resultList;
     }
@@ -148,7 +155,7 @@ public class DagHelper {
      * @param taskNodeList taskNodeList
      * @return task node list
      */
-    private static List<TaskNode> getFlowNodeListPre(TaskNode startNode, List<String> recoveryNodeNameList, List<TaskNode> taskNodeList) {
+    private static List<TaskNode> getFlowNodeListPre(TaskNode startNode, List<String> recoveryNodeNameList, List<TaskNode> taskNodeList, List<String> visitedNodeNameList) {
 
         List<TaskNode> resultList = new ArrayList<>();
 
@@ -158,16 +165,23 @@ public class DagHelper {
             resultList.add(startNode);
         }
         if (CollectionUtils.isEmpty(depList)) {
+            if (null != startNode) {
+                visitedNodeNameList.add(startNode.getName());
+            }
             return resultList;
         }
         for (String depNodeName : depList) {
             TaskNode start = findNodeByName(taskNodeList, depNodeName);
             if (recoveryNodeNameList.contains(depNodeName)) {
                 resultList.add(start);
-            } else {
+            } else if (!visitedNodeNameList.contains(depNodeName)) {
-                resultList.addAll(getFlowNodeListPre(start, recoveryNodeNameList, taskNodeList));
+                resultList.addAll(getFlowNodeListPre(start, recoveryNodeNameList, taskNodeList, visitedNodeNameList));
             }
         }
+        // why add (startNode != null) condition? for SonarCloud Quality Gate passed
+        if (null != startNode) {
+            visitedNodeNameList.add(startNode.getName());
+        }
         return resultList;
     }
 
@@ -369,7 +383,7 @@ public class DagHelper {
      */
     public static boolean haveConditionsAfterNode(String parentNodeName,
                                                   DAG<String, TaskNode, TaskNodeRelation> dag
-                                            ){
+    ){
         boolean result = false;
         Set<String> subsequentNodes = dag.getSubsequentNodes(parentNodeName);
         if(CollectionUtils.isEmpty(subsequentNodes)){

dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml

@@ -37,6 +37,16 @@
         order by id asc
     </select>
 
+    <select id="queryTopNProcessInstance" resultType="org.apache.dolphinscheduler.dao.entity.ProcessInstance">
+        select *
+        from t_ds_process_instance
+        where state = #{status}
+        and start_time between
+        #{startTime} and #{endTime}
+        order by end_time-start_time desc
+        limit #{size}
+    </select>
+
     <select id="queryByTenantIdAndStatus" resultType="org.apache.dolphinscheduler.dao.entity.ProcessInstance">
         select *
         from t_ds_process_instance

dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapper.xml

@@ -0,0 +1,103 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+<mapper namespace="org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper">
+    <select id="queryByName" resultType="org.apache.dolphinscheduler.dao.entity.WorkFlowLineage">
+        select tepd.id as work_flow_id,tepd.name as work_flow_name
+        from t_ds_process_definition tepd
+        left join t_ds_schedules tes on tepd.id = tes.process_definition_id
+        where tepd.project_id = #{projectId}
+        <if test="searchVal != null and searchVal != ''">
+            and  tepd.name like concat('%', #{searchVal}, '%')
+        </if>
+    </select>
+    <select id="queryByIds" resultType="org.apache.dolphinscheduler.dao.entity.WorkFlowLineage" databaseId="mysql">
+        select tepd.id as work_flow_id,tepd.name as work_flow_name,
+               (case when json_extract(tepd.process_definition_json, '$**.dependItemList') is not null then 1 else 0 end) as is_depend_work_flow,
+                          json_extract(tepd.process_definition_json, '$**.definitionId') as source_work_flow_id,
+                          tepd.release_state as work_flow_publish_status,
+                          tes.start_time as schedule_start_time,
+                          tes.end_time as schedule_end_time,
+                          tes.crontab as crontab,
+                          tes.release_state as schedule_publish_status
+        from t_ds_process_definition tepd
+        left join t_ds_schedules tes on tepd.id = tes.process_definition_id
+        where tepd.project_id = #{projectId}
+        <if test="ids != null and ids.size()>0">
+            and  tepd.id in
+            <foreach collection="ids" index="index" item="i" open="(" separator="," close=")">
+                #{i}
+            </foreach>
+        </if>
+    </select>
+
+    <select id="queryByIds" resultType="org.apache.dolphinscheduler.dao.entity.WorkFlowLineage" databaseId="pg">
+        select a.work_flow_id,
+               a.work_flow_name,
+               a.is_depend_work_flow,
+               array_agg(a.source_id) as source_id,
+               a.work_flow_publish_status,
+               a.schedule_start_time,
+               a.schedule_end_time,
+               a.crontab,
+               a.schedule_publish_status
+         from (
+               select tepd.id as work_flow_id,tepd.name as work_flow_name,
+                      case when tepd.process_definition_json::json#>'{tasks,1,dependence}' is not null then 1 else 0 end as is_depend_work_flow,
+                      (json_array_elements(tepd.process_definition_json::json#>'{tasks}')#>>'{dependence,dependTaskList,0,dependItemList,0,definitionId}') as source_id,
+                      tepd.release_state as work_flow_publish_status,
+                      tes.start_time as schedule_start_time,
+                      tes.end_time as schedule_end_time,
+                      tes.crontab as crontab,
+                      tes.release_state as schedule_publish_status
+                 from t_ds_process_definition tepd
+                 left join t_ds_schedules tes on tepd.id = tes.process_definition_id
+                 where tepd.project_id = #{projectId}
+                 <if test="ids != null and ids.size()>0">
+                     and  tepd.id in
+                     <foreach collection="ids" index="index" item="i" open="(" separator="," close=")">
+                     #{i}
+                     </foreach>
+                </if>
+             ) a
+        where (a.is_depend_work_flow = 1 and source_id is not null) or (a.is_depend_work_flow = 0)
+        group by a.work_flow_id,a.work_flow_name,a.is_depend_work_flow,a.work_flow_publish_status,a.schedule_start_time,
+                 a.schedule_end_time,a.crontab,a.schedule_publish_status
+    </select>
+
+    <select id="querySourceTarget" resultType="org.apache.dolphinscheduler.dao.entity.WorkFlowRelation" databaseId="mysql">
+        select id as target_work_flow_id,#{id} as source_work_flow_id
+        from t_ds_process_definition t
+        where json_extract(t.process_definition_json, '$**.dependItemList') is not null
+          and find_in_set(#{id}, replace(replace(replace(json_extract(t.process_definition_json, '$**.definitionId'), '[', ''),']', ''), ' ', '')) > 0
+    </select>
+
+    <select id="querySourceTarget" resultType="org.apache.dolphinscheduler.dao.entity.WorkFlowRelation" databaseId="pg">
+        select a.work_flow_id as target_work_flow_id,
+               a.source_id as source_work_flow_id
+         from (
+               select tepd.id as work_flow_id,
+                      (json_array_elements(tepd.process_definition_json::json#>'{tasks}')#>>'{dependence,dependTaskList,0,dependItemList,0,definitionId}') as source_id
+                 from t_ds_process_definition tepd
+                 left join t_ds_schedules tes on tepd.id = tes.process_definition_id
+                where tepd.project_id = 1) a
+        where source_id = #{id}::text;
+    </select>
+
+</mapper>

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java

@@ -17,6 +17,8 @@
 package org.apache.dolphinscheduler.dao.datasource;
 
 import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.DbType;
+import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -112,4 +114,51 @@ public class BaseDataSourceTest {
 
 
   }
+
+  @Test
+  public void testGetPassword() {
+    BaseDataSource dataSource = new BaseDataSource() {
+      @Override
+      public String driverClassSelector() {
+        return null;
+      }
+
+      @Override
+      public DbType dbTypeSelector() {
+        return null;
+      }
+    };
+
+    String password= "";
+    dataSource.setPassword(password);
+    Assert.assertEquals("", dataSource.getPassword());
+    password= "IUAjJCVeJipNVEl6TkRVMg==";
+    dataSource.setPassword(password);
+    Assert.assertNotNull(dataSource.getPassword());
+    Assert.assertNotNull(dataSource.getPassword());
+
+    dataSource.setPassword(password);
+    PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
+    Assert.assertEquals("123456", dataSource.getPassword());
+
+    dataSource.setPassword(password);
+    Assert.assertEquals("123456", dataSource.getPassword());
+    Assert.assertEquals("123456", dataSource.getPassword());
+    Assert.assertEquals("123456", dataSource.getPassword());
+
+    dataSource.setPassword(password);
+    PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
+    Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
+
+    dataSource.setPassword(password);
+    Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
+    Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
+    Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
+
+
+  }
+
+
+
+
 }

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSourceTest.java

@@ -16,6 +16,8 @@
  */
 package org.apache.dolphinscheduler.dao.datasource;
 
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -46,6 +48,7 @@ public class MySQLDataSourceTest {
         Assert.assertEquals("test_pwd?", dataSource.getPassword());
     }
 
+
     @Test
     public void testFilterOther(){
         MySQLDataSource dataSource = new MySQLDataSource();
@@ -61,4 +64,36 @@ public class MySQLDataSourceTest {
         other = dataSource.filterOther("serverTimezone=Asia/Shanghai&autoDeserialize=true&characterEncoding=utf8");
         Assert.assertEquals("serverTimezone=Asia/Shanghai&characterEncoding=utf8", other);
     }
+
+    @Test
+    public void testGetPasswordWithDecodePassword(){
+        MySQLDataSource dataSource = new MySQLDataSource();
+        String password= "";
+        dataSource.setPassword(password);
+        Assert.assertEquals("", dataSource.getPassword());
+        password= "IUAjJCVeJipNVEl6TkRVMg==";
+        dataSource.setPassword(password);
+        Assert.assertNotNull(dataSource.getPassword());
+        Assert.assertNotNull(dataSource.getPassword());
+
+        dataSource.setPassword(password);
+        PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
+        Assert.assertEquals("123456", dataSource.getPassword());
+
+        dataSource.setPassword(password);
+        Assert.assertEquals("123456", dataSource.getPassword());
+        Assert.assertEquals("123456", dataSource.getPassword());
+        Assert.assertEquals("123456", dataSource.getPassword());
+
+        dataSource.setPassword(password);
+        PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
+        Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
+
+        dataSource.setPassword(password);
+        Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
+        Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
+        Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
+
+    }
+
 }

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/UdfFuncTest.java

@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.dao.entity;
+
+import org.apache.dolphinscheduler.dao.entity.UdfFunc.UdfFuncDeserializer;
+import java.io.IOException;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class UdfFuncTest {
+
+  /**
+   * test to String
+   */
+  @Test
+  public void testUdfFuncToString() {
+    
+    UdfFunc udfFunc = new UdfFunc();
+    udfFunc.setResourceName("dolphin_resource_update");
+    udfFunc.setResourceId(2);
+    udfFunc.setClassName("org.apache.dolphinscheduler.test.mrUpdate");
+
+    Assert.assertEquals("{\"id\":0,\"userId\":0,\"funcName\":null,\"className\":\"org.apache.dolphinscheduler.test.mrUpdate\",\"argTypes\":null,\"database\":null,"
+            + "\"description\":null,\"resourceId\":2,\"resourceName\":\"dolphin_resource_update\",\"type\":null,\"createTime\":null,\"updateTime\":null}"
+        , udfFunc.toString());
+  }
+
+  /**
+   * test UdfFuncDeserializer.deserializeKey
+   *
+   * @throws IOException
+   */
+  @Test
+  public void testUdfFuncDeserializer() throws IOException {
+
+    // UdfFuncDeserializer.deserializeKey key is null
+    UdfFuncDeserializer udfFuncDeserializer = new UdfFuncDeserializer();
+    Assert.assertNull(udfFuncDeserializer.deserializeKey(null, null));
+
+    //
+    UdfFunc udfFunc = new UdfFunc();
+    udfFunc.setResourceName("dolphin_resource_update");
+    udfFunc.setResourceId(2);
+    udfFunc.setClassName("org.apache.dolphinscheduler.test.mrUpdate");
+
+    Assert.assertNotNull(udfFuncDeserializer.deserializeKey(udfFunc.toString(), null));
+  }
+
+}

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java

@@ -63,7 +63,6 @@ public class AccessTokenMapperTest {
         Integer userId = 1;
 
         AccessToken accessToken = createAccessToken(userId);
-        assertNotNull(accessToken.getId());
         assertThat(accessToken.getId(), greaterThan(0));
     }
 

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java

@@ -54,7 +54,6 @@ public class AlertMapperTest {
     @Test
     public void testInsert(){
         Alert expectedAlert = createAlert();
-        assertNotNull(expectedAlert.getId());
         assertThat(expectedAlert.getId(), greaterThan(0));
     }
 

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java

@@ -62,7 +62,6 @@ public class CommandMapperTest {
     @Test
     public void testInsert(){
         Command command = createCommand();
-        assertNotNull(command.getId());
         assertThat(command.getId(),greaterThan(0));
     }
 

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java

@@ -71,7 +71,6 @@ public class DataSourceMapperTest {
     @Test
     public void testInsert(){
         DataSource dataSource = createDataSource();
-        assertNotNull(dataSource.getId());
         assertThat(dataSource.getId(), greaterThan(0));
     }
 

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java

@@ -52,6 +52,25 @@ public class ProcessInstanceMapperTest {
     ProjectMapper projectMapper;
 
 
+    /**
+     * insert process instance with specified start time and end time,set state to SUCCESS
+     *
+     * @param startTime
+     * @param endTime
+     * @return
+     */
+    private ProcessInstance insertOne(Date startTime, Date endTime) {
+        ProcessInstance processInstance = new ProcessInstance();
+        Date start = startTime;
+        Date end = endTime;
+        processInstance.setStartTime(start);
+        processInstance.setEndTime(end);
+        processInstance.setState(ExecutionStatus.SUCCESS);
+
+        processInstanceMapper.insert(processInstance);
+        return processInstance;
+    }
+
     /**
      * insert
      * @return ProcessInstance
@@ -113,7 +132,7 @@ public class ProcessInstanceMapperTest {
         processInstanceMapper.updateById(processInstance);
 
         ProcessInstance processInstance1 = processInstanceMapper.queryDetailById(processInstance.getId());
-        Assert.assertNotEquals(processInstance1, 50);
+        Assert.assertNotNull(processInstance1);
         processInstanceMapper.deleteById(processInstance.getId());
     }
 
@@ -124,11 +143,11 @@ public class ProcessInstanceMapperTest {
     public void testQueryByHostAndStates() {
         ProcessInstance processInstance = insertOne();
         processInstance.setHost("192.168.2.155");
-        processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
+        processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
         processInstanceMapper.updateById(processInstance);
 
         int[] stateArray = new int[]{
-                ExecutionStatus.RUNNING_EXEUTION.ordinal(),
+                ExecutionStatus.RUNNING_EXECUTION.ordinal(),
                 ExecutionStatus.SUCCESS.ordinal()};
 
         List<ProcessInstance> processInstances = processInstanceMapper.queryByHostAndStatus(null, stateArray);
@@ -145,7 +164,7 @@ public class ProcessInstanceMapperTest {
 
 
         int[] stateArray = new int[]{
-                ExecutionStatus.RUNNING_EXEUTION.ordinal(),
+                ExecutionStatus.RUNNING_EXECUTION.ordinal(),
                 ExecutionStatus.SUCCESS.ordinal()};
 
         ProcessDefinition processDefinition = new ProcessDefinition();
@@ -155,7 +174,7 @@ public class ProcessInstanceMapperTest {
 
         ProcessInstance processInstance = insertOne();
         processInstance.setProcessDefinitionId(processDefinition.getId());
-        processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
+        processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
         processInstance.setIsSubProcess(Flag.NO);
         processInstance.setStartTime(new Date());
 
@@ -188,12 +207,12 @@ public class ProcessInstanceMapperTest {
     public void testSetFailoverByHostAndStateArray() {
 
         int[] stateArray = new int[]{
-                ExecutionStatus.RUNNING_EXEUTION.ordinal(),
+                ExecutionStatus.RUNNING_EXECUTION.ordinal(),
                 ExecutionStatus.SUCCESS.ordinal()};
 
         ProcessInstance processInstance = insertOne();
 
-        processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
+        processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
         processInstance.setHost("192.168.2.220");
         processInstanceMapper.updateById(processInstance);
         String host = processInstance.getHost();
@@ -214,9 +233,9 @@ public class ProcessInstanceMapperTest {
 
         ProcessInstance processInstance = insertOne();
 
-        processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
+        processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
         processInstanceMapper.updateById(processInstance);
-        processInstanceMapper.updateProcessInstanceByState(ExecutionStatus.RUNNING_EXEUTION, ExecutionStatus.SUCCESS);
+        processInstanceMapper.updateProcessInstanceByState(ExecutionStatus.RUNNING_EXECUTION, ExecutionStatus.SUCCESS);
 
         ProcessInstance processInstance1 = processInstanceMapper.selectById(processInstance.getId());
 
@@ -294,11 +313,11 @@ public class ProcessInstanceMapperTest {
     @Test
     public void testQueryLastRunningProcess() {
         ProcessInstance processInstance = insertOne();
-        processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
+        processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
         processInstanceMapper.updateById(processInstance);
 
         int[] stateArray = new int[]{
-                ExecutionStatus.RUNNING_EXEUTION.ordinal(),
+                ExecutionStatus.RUNNING_EXECUTION.ordinal(),
                 ExecutionStatus.SUBMITTED_SUCCESS.ordinal()};
 
         ProcessInstance processInstance1 = processInstanceMapper.queryLastRunningProcess(processInstance.getProcessDefinitionId(), null, null , stateArray);
@@ -329,4 +348,50 @@ public class ProcessInstanceMapperTest {
         processInstanceMapper.deleteById(processInstance.getId());
 
     }
+
+
+    /**
+     * test whether it is in descending order by running duration
+     *
+     * @param processInstances
+     * @return
+     */
+    private boolean isSortedByDuration(List<ProcessInstance> processInstances) {
+        for (int i = 1; i < processInstances.size(); i++) {
+            long d1 = processInstances.get(i).getEndTime().getTime() - processInstances.get(i).getStartTime().getTime();
+            long d2 = processInstances.get(i - 1).getEndTime().getTime() - processInstances.get(i - 1).getStartTime().getTime();
+            if (d1 > d2) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * test query top n process instance order by running duration
+     */
+    @Test
+    public void testQueryTopNProcessInstance() {
+        Date startTime1 = new Date(2019, 7, 9, 10, 9, 9);
+        Date endTime1 = new Date(2019, 7, 9, 10, 9, 14);
+        Date startTime2 = new Date(2020, 7, 9, 10, 9, 9);
+        Date endTime2 = new Date(2020, 7, 9, 10, 9, 30);
+        Date startTime3 = new Date(2020, 6, 9, 10, 9, 9);
+        Date endTime3 = new Date(2020, 7, 9, 10, 9, 30);
+        ProcessInstance processInstance1 = insertOne(startTime1, endTime1);
+        ProcessInstance processInstance2 = insertOne(startTime2, endTime2);
+        ProcessInstance processInstance3 = insertOne(startTime3, endTime3);
+        Date start = new Date(2020, 1, 1, 1, 1, 1);
+        Date end = new Date(2021, 1, 1, 1, 1, 1);
+        List<ProcessInstance> processInstances = processInstanceMapper.queryTopNProcessInstance(2, start, end,ExecutionStatus.SUCCESS);
+        Assert.assertEquals(2, processInstances.size());
+        Assert.assertTrue(isSortedByDuration(processInstances));
+        for (ProcessInstance processInstance : processInstances) {
+            Assert.assertTrue(processInstance.getState().typeIsSuccess());
+        }
+        processInstanceMapper.deleteById(processInstance1.getId());
+        processInstanceMapper.deleteById(processInstance2.getId());
+        processInstanceMapper.deleteById(processInstance3.getId());
+
+    }
 }

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java

@@ -64,7 +64,7 @@ public class TaskInstanceMapperTest {
         TaskInstance taskInstance = new TaskInstance();
         taskInstance.setFlag(Flag.YES);
         taskInstance.setName("ut task");
-        taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
+        taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
         taskInstance.setStartTime(new Date());
         taskInstance.setEndTime(new Date());
         taskInstance.setTaskJson("{}");
@@ -118,7 +118,7 @@ public class TaskInstanceMapperTest {
         taskInstanceMapper.updateById(task);
         List<Integer> taskInstances = taskInstanceMapper.queryTaskByProcessIdAndState(
                 task.getProcessInstanceId(),
-                ExecutionStatus.RUNNING_EXEUTION.ordinal()
+                ExecutionStatus.RUNNING_EXECUTION.ordinal()
         );
         taskInstanceMapper.deleteById(task.getId());
         Assert.assertNotEquals(taskInstances.size(), 0);
@@ -162,7 +162,7 @@ public class TaskInstanceMapperTest {
         taskInstanceMapper.updateById(task);
 
         List<TaskInstance> taskInstances = taskInstanceMapper.queryByHostAndStatus(
-                task.getHost(), new int[]{ExecutionStatus.RUNNING_EXEUTION.ordinal()}
+                task.getHost(), new int[]{ExecutionStatus.RUNNING_EXECUTION.ordinal()}
         );
         taskInstanceMapper.deleteById(task.getId());
         Assert.assertNotEquals(taskInstances.size(), 0);
@@ -179,7 +179,7 @@ public class TaskInstanceMapperTest {
 
         int setResult = taskInstanceMapper.setFailoverByHostAndStateArray(
                 task.getHost(),
-                new int[]{ExecutionStatus.RUNNING_EXEUTION.ordinal()},
+                new int[]{ExecutionStatus.RUNNING_EXECUTION.ordinal()},
                 ExecutionStatus.NEED_FAULT_TOLERANCE
         );
         taskInstanceMapper.deleteById(task.getId());
@@ -268,7 +268,7 @@ public class TaskInstanceMapperTest {
 
         ProcessInstance processInstance = new ProcessInstance();
         processInstance.setProcessDefinitionId(definition.getId());
-        processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
+        processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
         processInstance.setName("ut process");
         processInstance.setStartTime(new Date());
         processInstance.setEndTime(new Date());

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapperTest.java

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.dao.mapper;
+
+import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage;
+import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.annotation.Rollback;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest
+@Transactional
+@Rollback(true)
+public class WorkFlowLineageMapperTest {
+    @Autowired
+    private WorkFlowLineageMapper workFlowLineageMapper;
+
+    @Test
+    public void testQueryByName() {
+        List<WorkFlowLineage> workFlowLineages = workFlowLineageMapper.queryByName("test",1);
+        Assert.assertNotEquals(workFlowLineages.size(), 0);
+    }
+
+
+    @Test
+    public void testQueryByIds() {
+        Set<Integer> ids = new HashSet<>();
+        ids.add(1);
+        List<WorkFlowLineage> workFlowLineages = workFlowLineageMapper.queryByIds(ids,1);
+        Assert.assertNotEquals(workFlowLineages.size(), 0);
+    }
+
+    @Test
+    public void testQuerySourceTarget() {
+        List<WorkFlowRelation> workFlowRelations = workFlowLineageMapper.querySourceTarget(1);
+        Assert.assertNotEquals(workFlowRelations.size(), 0);
+    }
+}

dolphinscheduler-dist/pom.xml

@@ -318,6 +318,7 @@
                                                 <include>start-all.sh</include>
                                                 <include>stop-all.sh</include>
                                                 <include>dolphinscheduler-daemon.sh</include>
+                                                <include>status-all.sh</include>
                                             </includes>
                                         </source>
                                     </sources>

dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml

@@ -189,6 +189,7 @@
 				<include>start-all.sh</include>
 				<include>stop-all.sh</include>
 				<include>dolphinscheduler-daemon.sh</include>
+				<include>status-all.sh</include>
 			</includes>
 			<outputDirectory>./bin</outputDirectory>
 		</fileSet>

dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml

@@ -197,6 +197,7 @@
 				<include>start-all.sh</include>
 				<include>stop-all.sh</include>
 				<include>dolphinscheduler-daemon.sh</include>
+				<include>status-all.sh</include>
 			</includes>
 			<outputDirectory>./bin</outputDirectory>
 		</fileSet>

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/SQLTaskExecutionContext.java

@@ -18,7 +18,8 @@
 package org.apache.dolphinscheduler.server.entity;
 
 import org.apache.dolphinscheduler.dao.entity.UdfFunc;
-
+import org.apache.dolphinscheduler.dao.entity.UdfFunc.UdfFuncDeserializer;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import java.io.Serializable;
 import java.util.Map;
 
@@ -40,6 +41,7 @@ public class SQLTaskExecutionContext implements Serializable {
     /**
      * udf function tenant code map
      */
+    @JsonDeserialize(keyUsing = UdfFuncDeserializer.class)
     private Map<UdfFunc,String> udfFuncTenantCodeMap;
 
 

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java

@@ -123,8 +123,8 @@ public class MasterServer {
         // self tolerant
         this.zkMasterClient.start();
 
-        //
+        // scheduler start
-        masterSchedulerService.start();
+        this.masterSchedulerService.start();
 
         // start QuartzExecutors
         // what system should do if exception

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java

@@ -89,6 +89,7 @@ public class TaskResponseService {
             eventQueue.put(taskResponseEvent);
         } catch (InterruptedException e) {
             logger.error("put task : {} error :{}", taskResponseEvent,e);
+            Thread.currentThread().interrupt();
         }
     }
 

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java

@@ -124,7 +124,7 @@ public class ConditionsTaskExecThread extends MasterBaseTaskExecThread {
     private void initTaskParameters() {
         this.taskInstance.setLogPath(getTaskLogPath(taskInstance));
         this.taskInstance.setHost(NetUtils.getHost() + Constants.COLON + masterConfig.getListenPort());
-        taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
+        taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
         taskInstance.setStartTime(new Date());
         this.processService.saveTaskInstance(taskInstance);
 

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java

@@ -173,7 +173,7 @@ public class DependentTaskExecThread extends MasterBaseTaskExecThread {
     private void initTaskParameters() {
         taskInstance.setLogPath(getTaskLogPath(taskInstance));
         taskInstance.setHost(NetUtils.getHost() + Constants.COLON + masterConfig.getListenPort());
-        taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
+        taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
         taskInstance.setStartTime(new Date());
         processService.updateTaskInstance(taskInstance);
     }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java

@@ -172,7 +172,7 @@ public class MasterBaseTaskExecThread implements Callable<Boolean> {
                 return true;
             }
             // task cannot submit when running
-            if(taskInstance.getState() == ExecutionStatus.RUNNING_EXEUTION){
+            if(taskInstance.getState() == ExecutionStatus.RUNNING_EXECUTION){
                 logger.info(String.format("submit to task, but task [%s] state already be running. ", taskInstance.getName()));
                 return true;
             }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java

@@ -35,7 +35,6 @@ import org.apache.dolphinscheduler.dao.utils.DagHelper;
 import org.apache.dolphinscheduler.remote.NettyRemotingClient;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.utils.AlertManager;
-import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
 import org.slf4j.Logger;
@@ -123,12 +122,12 @@ public class MasterExecThread implements Runnable {
     /**
      * alert manager
      */
-    private AlertManager alertManager = new AlertManager();
+    private AlertManager alertManager;
 
     /**
      * the object of DAG
      */
-    private DAG<String,TaskNode,TaskNodeRelation> dag;
+    private DAG<String, TaskNode, TaskNodeRelation> dag;
 
     /**
      *  process service
@@ -151,15 +150,20 @@ public class MasterExecThread implements Runnable {
      * @param processService processService
      * @param nettyRemotingClient nettyRemotingClient
      */
-    public MasterExecThread(ProcessInstance processInstance, ProcessService processService, NettyRemotingClient nettyRemotingClient){
+    public MasterExecThread(ProcessInstance processInstance
+            , ProcessService processService
+            , NettyRemotingClient nettyRemotingClient
+            , AlertManager alertManager
+            , MasterConfig masterConfig) {
         this.processService = processService;
 
         this.processInstance = processInstance;
-        this.masterConfig = SpringApplicationContext.getBean(MasterConfig.class);
+        this.masterConfig = masterConfig;
         int masterTaskExecNum = masterConfig.getMasterExecTaskNum();
         this.taskExecService = ThreadUtils.newDaemonFixedThreadExecutor("Master-Task-Exec-Thread",
                 masterTaskExecNum);
         this.nettyRemotingClient = nettyRemotingClient;
+        this.alertManager = alertManager;
     }
 
 
@@ -248,6 +252,9 @@ public class MasterExecThread implements Runnable {
         }
 
         while(Stopper.isRunning()){
+
+            logger.info("process {} start to complement {} data",
+                    processInstance.getId(), DateUtils.dateToString(scheduleDate));
             // prepare dag and other info
             prepareProcess();
 
@@ -262,13 +269,13 @@ public class MasterExecThread implements Runnable {
             // execute process ,waiting for end
             runProcess();
 
+            endProcess();
             // process instance failure ，no more complements
             if(!processInstance.getState().typeIsSuccess()){
                 logger.info("process {} state {}, complement not completely!",
                         processInstance.getId(), processInstance.getState());
                 break;
             }
-
             //  current process instance success ,next execute
             if(null == iterator){
                 // loop by day
@@ -287,9 +294,7 @@ public class MasterExecThread implements Runnable {
                 }
                 scheduleDate = iterator.next();
             }
-
+            // flow end
-            logger.info("process {} start to complement {} data",
-                    processInstance.getId(), DateUtils.dateToString(scheduleDate));
             // execute next process instance complement data
             processInstance.setScheduleTime(scheduleDate);
             if(cmdParam.containsKey(Constants.CMDPARAM_RECOVERY_START_NODE_STRING)){
@@ -297,23 +302,16 @@ public class MasterExecThread implements Runnable {
                 processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam));
             }
 
-            List<TaskInstance> taskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId());
+            processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
-            for(TaskInstance taskInstance : taskInstanceList){
-                taskInstance.setFlag(Flag.NO);
-                processService.updateTaskInstance(taskInstance);
-            }
-            processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
             processInstance.setGlobalParams(ParameterUtils.curingGlobalParams(
                     processInstance.getProcessDefinition().getGlobalParamMap(),
                     processInstance.getProcessDefinition().getGlobalParamList(),
                     CommandType.COMPLEMENT_DATA, processInstance.getScheduleTime()));
-
+            processInstance.setId(0);
+            processInstance.setStartTime(new Date());
+            processInstance.setEndTime(null);
             processService.saveProcessInstance(processInstance);
         }
-
-        // flow end
-        endProcess();
-
     }
 
 
@@ -727,7 +725,7 @@ public class MasterExecThread implements Runnable {
             // if the running task is not completed, the state remains unchanged
             return state;
         }else{
-            return ExecutionStatus.RUNNING_EXEUTION;
+            return ExecutionStatus.RUNNING_EXECUTION;
         }
     }
 
@@ -804,7 +802,7 @@ public class MasterExecThread implements Runnable {
         ProcessInstance instance = processService.findProcessInstanceById(processInstance.getId());
         ExecutionStatus state = instance.getState();
 
-        if(activeTaskNode.size() > 0 || retryTaskExists()){
+        if(activeTaskNode.size() > 0 || hasRetryTaskInStandBy()){
             // active task and retry task exists
             return runningState(state);
         }
@@ -837,11 +835,11 @@ public class MasterExecThread implements Runnable {
         }
 
         // success
-        if(state == ExecutionStatus.RUNNING_EXEUTION){
+        if(state == ExecutionStatus.RUNNING_EXECUTION){
             List<TaskInstance> killTasks = getCompleteTaskByState(ExecutionStatus.KILL);
             if(readyToSubmitTaskList.size() > 0){
                 //tasks currently pending submission, no retries, indicating that depend is waiting to complete
-                return ExecutionStatus.RUNNING_EXEUTION;
+                return ExecutionStatus.RUNNING_EXECUTION;
             }else if(CollectionUtils.isNotEmpty(killTasks)){
                 // tasks maybe killed manually
                 return ExecutionStatus.FAILURE;
@@ -854,24 +852,6 @@ public class MasterExecThread implements Runnable {
         return state;
     }
 
-    /**
-     * whether standby task list have retry tasks
-     * @return
-     */
-    private boolean retryTaskExists() {
-
-        boolean result = false;
-
-        for(String taskName : readyToSubmitTaskList.keySet()){
-            TaskInstance task = readyToSubmitTaskList.get(taskName);
-            if(task.getState().typeIsFailure()){
-                result = true;
-                break;
-            }
-        }
-        return result;
-    }
-
     /**
      * whether complement end
      * @return Boolean whether is complement end
@@ -1046,6 +1026,7 @@ public class MasterExecThread implements Runnable {
                 Thread.sleep(Constants.SLEEP_TIME_MILLIS);
             } catch (InterruptedException e) {
                 logger.error(e.getMessage(),e);
+                Thread.currentThread().interrupt();
             }
             updateProcessInstanceState();
         }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java

@@ -16,6 +16,11 @@
  */
 package org.apache.dolphinscheduler.server.master.runner;
 
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import javax.annotation.PostConstruct;
+
 import org.apache.curator.framework.imps.CuratorFrameworkState;
 import org.apache.curator.framework.recipes.locks.InterProcessMutex;
 import org.apache.dolphinscheduler.common.Constants;
@@ -28,6 +33,7 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
 import org.apache.dolphinscheduler.remote.NettyRemotingClient;
 import org.apache.dolphinscheduler.remote.config.NettyClientConfig;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
+import org.apache.dolphinscheduler.server.utils.AlertManager;
 import org.apache.dolphinscheduler.server.zk.ZKMasterClient;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 import org.slf4j.Logger;
@@ -35,10 +41,6 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import javax.annotation.PostConstruct;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
 /**
  *  master scheduler thread
  */
@@ -68,6 +70,11 @@ public class MasterSchedulerService extends Thread {
     @Autowired
     private MasterConfig masterConfig;
 
+    /**
+     * alert manager
+     */
+    private AlertManager alertManager = new AlertManager();
+
     /**
      *  netty remoting client
      */
@@ -90,7 +97,7 @@ public class MasterSchedulerService extends Thread {
     }
 
     @Override
-    public void start(){
+    public synchronized void start(){
         super.setName("MasterSchedulerService");
         super.start();
     }
@@ -100,7 +107,9 @@ public class MasterSchedulerService extends Thread {
         boolean terminated = false;
         try {
             terminated = masterExecService.awaitTermination(5, TimeUnit.SECONDS);
-        } catch (InterruptedException ignore) {}
+        } catch (InterruptedException ignore) {
+            Thread.currentThread().interrupt();
+        }
         if(!terminated){
             logger.warn("masterExecService shutdown without terminated, increase await time");
         }
@@ -139,7 +148,13 @@ public class MasterSchedulerService extends Thread {
                                     this.masterConfig.getMasterExecThreads() - activeCount, command);
                             if (processInstance != null) {
                                 logger.info("start master exec thread , split DAG ...");
-                                masterExecService.execute(new MasterExecThread(processInstance, processService, nettyRemotingClient));
+                                masterExecService.execute(
+                                        new MasterExecThread(
+                                                processInstance
+                                                , processService
+                                                , nettyRemotingClient
+                                                , alertManager
+                                                , masterConfig));
                             }
                         }catch (Exception e){
                             logger.error("scan command error ", e);

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java

@@ -93,7 +93,7 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread {
             return false;
         }
 
-        taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
+        taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
         taskInstance.setStartTime(new Date());
         processService.updateTaskInstance(taskInstance);
         return true;

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java

@@ -167,7 +167,7 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
     private TaskExecuteAckCommand buildAckCommand(TaskExecutionContext taskExecutionContext) {
         TaskExecuteAckCommand ackCommand = new TaskExecuteAckCommand();
         ackCommand.setTaskInstanceId(taskExecutionContext.getTaskInstanceId());
-        ackCommand.setStatus(ExecutionStatus.RUNNING_EXEUTION.getCode());
+        ackCommand.setStatus(ExecutionStatus.RUNNING_EXECUTION.getCode());
         ackCommand.setLogPath(getTaskLogPath(taskExecutionContext));
         ackCommand.setHost(taskExecutionContext.getHost());
         ackCommand.setStartTime(new Date());

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java

@@ -50,6 +50,8 @@ public class TaskManager {
     switch (EnumUtils.getEnum(TaskType.class,taskExecutionContext.getTaskType())) {
         case SHELL:
         return new ShellTask(taskExecutionContext, logger);
+      case WATERDROP:
+        return new ShellTask(taskExecutionContext, logger);
       case PROCEDURE:
         return new ProcedureTask(taskExecutionContext, logger);
       case SQL:

dolphinscheduler-server/src/main/resources/config/install_config.conf

@@ -38,6 +38,9 @@ dbname="dolphinscheduler"
 # zk cluster
 zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
 
+# zk root directory
+zkRoot="/dolphinscheduler"
+
 # Note: the target installation path for dolphinscheduler, please not config as the same as the current path (pwd)
 installPath="/data1_1T/dolphinscheduler"
 

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/entity/SQLTaskExecutionContextTest.java

@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.entity;
+
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.dao.entity.UdfFunc;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.Test;
+
+public class SQLTaskExecutionContextTest {
+
+  /**
+   * test parse  josn String to TaskExecutionContext
+   */
+  @Test
+  public void testTaskExecutionContext() {
+    String contextJson = "{\n"
+        + "    \"taskInstanceId\":32,\n"
+        + "    \"taskName\":\"test-hive-func\",\n"
+        + "    \"startTime\":\"2020-07-19 16:45:46\",\n"
+        + "    \"taskType\":\"SQL\",\n"
+        + "    \"host\":null,\n"
+        + "    \"executePath\":\"/tmp/dolphinscheduler/exec/process/1/5/14/32\",\n"
+        + "    \"logPath\":null,\n"
+        + "    \"taskJson\":\"{\\\"id\\\":\\\"tasks-70999\\\",\\\"name\\\":\\\"test-hive-func\\\""
+        + ",\\\"desc\\\":null,\\\"type\\\":\\\"SQL\\\",\\\"runFlag\\\":\\\"NORMAL\\\","
+        + "\\\"loc\\\":null,\\\"maxRetryTimes\\\":0,\\\"retryInterval\\\":1,"
+        + "\\\"params\\\":{\\\"type\\\":\\\"HIVE\\\",\\\"datasource\\\":2,"
+        + "\\\"sql\\\":\\\"select mid_id, user_id,"
+        + " version_code, version_name, lang, source, os, area, model, "
+        + "brand, sdk_version, gmail, height_width, app_time, network,"
+        + " lng, lat, dt,\\\\n       Lower(model)\\\\nfrom dws_uv_detail_day limit 5;"
+        + "\\\",\\\"udfs\\\":\\\"1\\\",\\\"sqlType\\\":\\\"0\\\",\\\"title\\\":\\\""
+        + "test-hive-user-func\\\",\\\"receivers\\\":\\\"534634799@qq.com\\\","
+        + "\\\"receiversCc\\\":\\\"\\\",\\\"showType\\\":\\\"TABLE\\\",\\\"localParams\\\":[],"
+        + "\\\"connParams\\\":\\\"\\\",\\\"preStatements\\\":[],\\\"postStatements\\\":[]},"
+        + "\\\"preTasks\\\":[],\\\"extras\\\":null,\\\"depList\\\":[],\\\"dependence\\\":{},"
+        + "\\\"conditionResult\\\":{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]},"
+        + "\\\"taskInstancePriority\\\":\\\"MEDIUM\\\",\\\"workerGroup\\\":\\\"default\\\","
+        + "\\\"workerGroupId\\\":null,\\\"timeout\\\":{\\\"strategy\\\":\\\"\\\",\\\"interval\\\":null,"
+        + "\\\"enable\\\":false},\\\"conditionsTask\\\":false,\\\"forbidden\\\":false,"
+        + "\\\"taskTimeoutParameter\\\":{\\\"enable\\\":false,\\\"strategy\\\":null,"
+        + "\\\"interval\\\":0}}\",\n"
+        + "    \"processId\":0,\n"
+        + "    \"appIds\":null,\n"
+        + "    \"processInstanceId\":14,\n"
+        + "    \"scheduleTime\":null,\n"
+        + "    \"globalParams\":null,\n"
+        + "    \"executorId\":2,\n"
+        + "    \"cmdTypeIfComplement\":2,\n"
+        + "    \"tenantCode\":\"sl\",\n"
+        + "    \"queue\":\"sl\",\n"
+        + "    \"processDefineId\":5,\n"
+        + "    \"projectId\":1,\n"
+        + "    \"taskParams\":null,\n"
+        + "    \"envFile\":null,\n"
+        + "    \"definedParams\":null,\n"
+        + "    \"taskAppId\":null,\n"
+        + "    \"taskTimeoutStrategy\":0,\n"
+        + "    \"taskTimeout\":0,\n"
+        + "    \"workerGroup\":\"default\",\n"
+        + "    \"resources\":{\n"
+        + "    },\n"
+        + "    \"sqlTaskExecutionContext\":{\n"
+        + "        \"warningGroupId\":0,\n"
+        + "        \"connectionParams\":\"{\\\"type\\\":null,\\\"address\\\":"
+        + "\\\"jdbc:hive2://localhost:10000\\\",\\\"database\\\":\\\"gmall\\\","
+        + "\\\"jdbcUrl\\\":\\\"jdbc:hive2://localhost:10000/gmall\\\","
+        + "\\\"user\\\":\\\"sl-test\\\",\\\"password\\\":\\\"123456sl\\\"}\",\n"
+        + "        \"udfFuncTenantCodeMap\": null"
+        + "    },\n"
+        + "    \"dataxTaskExecutionContext\":{\n"
+        + "        \"dataSourceId\":0,\n"
+        + "        \"sourcetype\":0,\n"
+        + "        \"sourceConnectionParams\":null,\n"
+        + "        \"dataTargetId\":0,\n"
+        + "        \"targetType\":0,\n"
+        + "        \"targetConnectionParams\":null\n"
+        + "    },\n"
+        + "    \"dependenceTaskExecutionContext\":null,\n"
+        + "    \"sqoopTaskExecutionContext\":{\n"
+        + "        \"dataSourceId\":0,\n"
+        + "        \"sourcetype\":0,\n"
+        + "        \"sourceConnectionParams\":null,\n"
+        + "        \"dataTargetId\":0,\n"
+        + "        \"targetType\":0,\n"
+        + "        \"targetConnectionParams\":null\n"
+        + "    },\n"
+        + "    \"procedureTaskExecutionContext\":{\n"
+        + "        \"connectionParams\":null\n"
+        + "    }\n"
+        + "}\n";
+
+    TaskExecutionContext taskExecutionContext = JSONUtils.parseObject(contextJson, TaskExecutionContext.class);
+
+    assertNotNull(taskExecutionContext);
+  }
+
+  @Test
+  public void testSqlTaskExecutionContext() {
+
+    SQLTaskExecutionContext sqlTaskExecutionContext = new SQLTaskExecutionContext();
+    sqlTaskExecutionContext.setWarningGroupId(0);
+
+    Map<UdfFunc, String> udfmap = new HashMap<>();
+
+    UdfFunc udfFunc = new UdfFunc();
+    udfFunc.setArgTypes("1");
+    udfFunc.setId(1);
+    udfFunc.setResourceName("name1");
+    udfmap.put(udfFunc, "map1");
+
+    UdfFunc udfFunc2 = new UdfFunc();
+    udfFunc2.setArgTypes("2");
+    udfFunc2.setId(2);
+    udfFunc2.setResourceName("name2");
+    udfmap.put(udfFunc2, "map2");
+
+    sqlTaskExecutionContext.setUdfFuncTenantCodeMap(udfmap);
+
+    String contextJson = JSONUtils.toJsonString(sqlTaskExecutionContext);
+    SQLTaskExecutionContext parseSqlTask = JSONUtils.parseObject(contextJson, SQLTaskExecutionContext.class);
+
+    assertNotNull(parseSqlTask);
+    assertEquals(sqlTaskExecutionContext.getWarningGroupId(), parseSqlTask.getWarningGroupId());
+    assertEquals(sqlTaskExecutionContext.getUdfFuncTenantCodeMap().size(), parseSqlTask.getUdfFuncTenantCodeMap().size());
+  }
+
+  /**
+   * test the SQLTaskExecutionContext
+   */
+  @Test
+  public void testSqlTaskExecutionContextParse() {
+
+    // SQLTaskExecutionContext.udfFuncTenantCodeMap is null
+    String contextJson = "{\n"
+        + "    \"warningGroupId\":0,\n"
+        + "    \"connectionParams\":null,\n"
+        + "    \"udfFuncTenantCodeMap\":null"
+        + "}\n}";
+    SQLTaskExecutionContext parseSqlTask = JSONUtils.parseObject(contextJson, SQLTaskExecutionContext.class);
+
+    assertNotNull(parseSqlTask);
+    assertEquals(0,parseSqlTask.getWarningGroupId());
+    assertNull(parseSqlTask.getUdfFuncTenantCodeMap());
+
+    // SQLTaskExecutionContext.udfFuncTenantCodeMap is not null
+    contextJson = "{\"warningGroupId\":0,"
+        + "\"connectionParams\":null,"
+        + "\"udfFuncTenantCodeMap\":{\""
+        + "{\\\"id\\\":2,\\\"userId\\\":0,"
+        + "\\\"funcName\\\":null,\\\"className\\\":null,\\\"argTypes\\\":\\\"2\\\",\\\"database\\\":null,"
+        + "\\\"description\\\":null,\\\"resourceId\\\":0,\\\"resourceName\\\":\\\"name2\\\",\\\"type\\\":null,"
+        + "\\\"createTime\\\":null,\\\"updateTime\\\":null}\":\"map2\","
+        + "\"{\\\"id\\\":1,\\\"userId\\\":0,\\\"funcName\\\":null,"
+        + "\\\"className\\\":null,\\\"argTypes\\\":\\\"1\\\","
+        + "\\\"database\\\":null,\\\"description\\\":null,"
+        + "\\\"resourceId\\\":0,\\\"resourceName\\\":\\\"name1\\\","
+        + "\\\"type\\\":null,\\\"createTime\\\":null,\\\"updateTime\\\":null}\":\"map1\"}}\n";
+
+    SQLTaskExecutionContext parseSqlTask2 = JSONUtils.parseObject(contextJson, SQLTaskExecutionContext.class);
+
+    assertNotNull(parseSqlTask2);
+    assertEquals(0,parseSqlTask2.getWarningGroupId());
+    assertEquals(2,parseSqlTask2.getUdfFuncTenantCodeMap().size());
+  }
+
+}

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java

@@ -124,7 +124,7 @@ public class ConditionsTaskTest {
         ProcessInstance processInstance = new ProcessInstance();
         processInstance.setId(10112);
         processInstance.setProcessDefinitionId(100001);
-        processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
+        processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
 
         return processInstance;
     }

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java

@@ -176,7 +176,7 @@ public class DependentTaskTest {
         ProcessInstance processInstance = new ProcessInstance();
         processInstance.setId(10111);
         processInstance.setProcessDefinitionId(0);
-        processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
+        processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
 
         return processInstance;
     }

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java

@@ -72,8 +72,6 @@ public class MasterExecThreadTest {
         applicationContext = mock(ApplicationContext.class);
         config = new MasterConfig();
         config.setMasterExecTaskNum(1);
-        SpringApplicationContext springApplicationContext = new SpringApplicationContext();
-        springApplicationContext.setApplicationContext(applicationContext);
         Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config);
 
         processInstance = mock(ProcessInstance.class);
@@ -84,14 +82,17 @@ public class MasterExecThreadTest {
         Mockito.when(processInstance.getScheduleTime()).thenReturn(DateUtils.stringToDate("2020-01-01 00:00:00"));
         Map<String, String> cmdParam = new HashMap<>();
         cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, "2020-01-01 00:00:00");
-        cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, "2020-01-31 23:00:00");
+        cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, "2020-01-20 23:00:00");
         Mockito.when(processInstance.getCommandParam()).thenReturn(JSONUtils.toJsonString(cmdParam));
         ProcessDefinition processDefinition = new ProcessDefinition();
         processDefinition.setGlobalParamMap(Collections.EMPTY_MAP);
         processDefinition.setGlobalParamList(Collections.EMPTY_LIST);
         Mockito.when(processInstance.getProcessDefinition()).thenReturn(processDefinition);
 
-        masterExecThread = PowerMockito.spy(new MasterExecThread(processInstance, processService,null));
+        masterExecThread = PowerMockito.spy(new MasterExecThread(
+                processInstance
+                , processService
+                ,null, null, config));
         // prepareProcess init dag
         Field dag = MasterExecThread.class.getDeclaredField("dag");
         dag.setAccessible(true);
@@ -114,11 +115,11 @@ public class MasterExecThreadTest {
             Method method = MasterExecThread.class.getDeclaredMethod("executeComplementProcess");
             method.setAccessible(true);
             method.invoke(masterExecThread);
-            // one create save, and 1-30 for next save, and last day 31 no save
+            // one create save, and 1-30 for next save, and last day 20 no save
-            verify(processService, times(31)).saveProcessInstance(processInstance);
+            verify(processService, times(20)).saveProcessInstance(processInstance);
         }catch (Exception e){
             e.printStackTrace();
-            Assert.assertTrue(false);
+            Assert.fail();
         }
     }
 
@@ -133,10 +134,10 @@ public class MasterExecThreadTest {
             Method method = MasterExecThread.class.getDeclaredMethod("executeComplementProcess");
             method.setAccessible(true);
             method.invoke(masterExecThread);
-            // one create save, and 15(1 to 31 step 2) for next save, and last day 31 no save
+            // one create save, and 9(1 to 20 step 2) for next save, and last day 31 no save
-            verify(processService, times(15)).saveProcessInstance(processInstance);
+            verify(processService, times(9)).saveProcessInstance(processInstance);
         }catch (Exception e){
-            Assert.assertTrue(false);
+            Assert.fail();
         }
     }
 

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java

@@ -17,11 +17,18 @@
 
 package org.apache.dolphinscheduler.server.master.consumer;
 
+import java.util.Date;
+
 import org.apache.dolphinscheduler.common.enums.CommandType;
 import org.apache.dolphinscheduler.common.enums.DbType;
 import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.common.enums.Priority;
-import org.apache.dolphinscheduler.dao.entity.*;
+import org.apache.dolphinscheduler.common.thread.Stopper;
+import org.apache.dolphinscheduler.dao.entity.DataSource;
+import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
+import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.dao.entity.Tenant;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.master.dispatch.ExecutorDispatcher;
 import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager;
@@ -32,8 +39,10 @@ import org.apache.dolphinscheduler.server.zk.SpringZKServer;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue;
+import org.apache.dolphinscheduler.service.zk.CuratorZookeeperClient;
 import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator;
 import org.apache.dolphinscheduler.service.zk.ZookeeperConfig;
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -42,11 +51,9 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.test.context.ContextConfiguration;
 import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
 
-import java.util.Date;
-
 
 @RunWith(SpringJUnit4ClassRunner.class)
-@ContextConfiguration(classes={DependencyConfig.class, SpringApplicationContext.class, SpringZKServer.class,
+@ContextConfiguration(classes={DependencyConfig.class, SpringApplicationContext.class, SpringZKServer.class, CuratorZookeeperClient.class,
         NettyExecutorManager.class, ExecutorDispatcher.class, ZookeeperRegistryCenter.class, TaskPriorityQueueConsumer.class,
         ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, MasterConfig.class})
 public class TaskPriorityQueueConsumerTest {
@@ -250,5 +257,11 @@ public class TaskPriorityQueueConsumerTest {
         taskPriorityQueueConsumer.taskInstanceIsFinalState(1);
     }
 
+    @After
+    public void close() {
+        Stopper.stop();
+    }
+
+
 
 }

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java

@@ -78,5 +78,7 @@ public class ExecutorDispatcherTest {
 
         ExecutionContext executionContext = ExecutionContextTestUtils.getExecutionContext(port);
         executorDispatcher.dispatch(executionContext);
+
+        workerRegistry.unRegistry();
     }
 }

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java

@@ -74,5 +74,6 @@ public class RoundRobinHostManagerTest {
         Host host = roundRobinHostManager.select(context);
         Assert.assertTrue(StringUtils.isNotEmpty(host.getAddress()));
         Assert.assertTrue(host.getAddress().equalsIgnoreCase(NetUtils.getHost() + ":" + workerConfig.getListenPort()));
+        workerRegistry.unRegistry();
     }
 }

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java

@@ -43,7 +43,7 @@ public class TaskResponseServiceTest {
 
     @Test
     public void testAdd(){
-        TaskResponseEvent taskResponseEvent = TaskResponseEvent.newAck(ExecutionStatus.RUNNING_EXEUTION, new Date(),
+        TaskResponseEvent taskResponseEvent = TaskResponseEvent.newAck(ExecutionStatus.RUNNING_EXECUTION, new Date(),
                 "", "", "", 1);
         taskResponseService.addResponse(taskResponseEvent);
         Assert.assertTrue(taskResponseService.getEventQueue().size() == 1);
@@ -57,7 +57,7 @@ public class TaskResponseServiceTest {
 
     @Test
     public void testStop(){
-        TaskResponseEvent taskResponseEvent = TaskResponseEvent.newAck(ExecutionStatus.RUNNING_EXEUTION, new Date(),
+        TaskResponseEvent taskResponseEvent = TaskResponseEvent.newAck(ExecutionStatus.RUNNING_EXECUTION, new Date(),
                 "", "", "", 1);
         taskResponseService.addResponse(taskResponseEvent);
         taskResponseService.stop();

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryTest.java

@@ -58,6 +58,7 @@ public class MasterRegistryTest {
         String masterNodePath = masterPath + "/" + (Constants.LOCAL_ADDRESS + ":" + masterConfig.getListenPort());
         String heartbeat = zookeeperRegistryCenter.getZookeeperCachedOperator().get(masterNodePath);
         Assert.assertEquals(HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH, heartbeat.split(",").length);
+        masterRegistry.unRegistry();
     }
 
     @Test

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java

@@ -17,65 +17,75 @@
 
 package org.apache.dolphinscheduler.server.master.runner;
 
+import java.util.HashSet;
+import java.util.Set;
+
 import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
-import org.apache.dolphinscheduler.server.master.consumer.TaskPriorityQueueConsumer;
-import org.apache.dolphinscheduler.server.master.dispatch.ExecutorDispatcher;
-import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager;
-import org.apache.dolphinscheduler.server.registry.DependencyConfig;
-import org.apache.dolphinscheduler.server.registry.ZookeeperNodeManager;
 import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter;
-import org.apache.dolphinscheduler.server.zk.SpringZKServer;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
 import org.apache.dolphinscheduler.service.process.ProcessService;
-import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator;
+import org.junit.Assert;
-import org.apache.dolphinscheduler.service.zk.ZookeeperConfig;
+import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.springframework.context.ApplicationContext;
-import org.springframework.test.context.ContextConfiguration;
-import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
 
-import java.util.HashSet;
+import com.google.common.collect.Sets;
-import java.util.Set;
 
-@RunWith(SpringJUnit4ClassRunner.class)
+@RunWith(MockitoJUnitRunner.Silent.class)
-@ContextConfiguration(classes={DependencyConfig.class, SpringApplicationContext.class, SpringZKServer.class,
+@PrepareForTest(MasterTaskExecThread.class)
-        NettyExecutorManager.class, ExecutorDispatcher.class, ZookeeperRegistryCenter.class, TaskPriorityQueueConsumer.class,
-        ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class})
 public class MasterTaskExecThreadTest {
 
+    private MasterTaskExecThread masterTaskExecThread;
+
+    private SpringApplicationContext springApplicationContext;
+
+    private ZookeeperRegistryCenter zookeeperRegistryCenter;
+
+    @Before
+    public void setUp() {
+
+        ApplicationContext applicationContext = PowerMockito.mock(ApplicationContext.class);
+        this.springApplicationContext = new SpringApplicationContext();
+        springApplicationContext.setApplicationContext(applicationContext);
+        this.zookeeperRegistryCenter = PowerMockito.mock(ZookeeperRegistryCenter.class);
+        PowerMockito.when(SpringApplicationContext.getBean(ZookeeperRegistryCenter.class))
+                .thenReturn(this.zookeeperRegistryCenter);
+        this.masterTaskExecThread = new MasterTaskExecThread(null);
+    }
 
     @Test
     public void testExistsValidWorkerGroup1(){
-        ZookeeperRegistryCenter zookeeperRegistryCenter = Mockito.mock(ZookeeperRegistryCenter.class);
+
-        Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(null);
+        Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(Sets.newHashSet());
-        MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(null);
+        boolean b = masterTaskExecThread.existsValidWorkerGroup("default");
-        masterTaskExecThread.existsValidWorkerGroup("default");
+        Assert.assertFalse(b);
     }
     @Test
     public void testExistsValidWorkerGroup2(){
-        ZookeeperRegistryCenter zookeeperRegistryCenter = Mockito.mock(ZookeeperRegistryCenter.class);
         Set<String> workerGorups = new HashSet<>();
         workerGorups.add("test1");
         workerGorups.add("test2");
 
         Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(workerGorups);
-        MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(null);
+        boolean b = masterTaskExecThread.existsValidWorkerGroup("default");
-        masterTaskExecThread.existsValidWorkerGroup("default");
+        Assert.assertFalse(b);
     }
 
     @Test
     public void testExistsValidWorkerGroup3(){
-        ZookeeperRegistryCenter zookeeperRegistryCenter = Mockito.mock(ZookeeperRegistryCenter.class);
         Set<String> workerGorups = new HashSet<>();
         workerGorups.add("test1");
 
         Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(workerGorups);
         Mockito.when(zookeeperRegistryCenter.getWorkerGroupNodesDirectly("test1")).thenReturn(workerGorups);
-        MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(null);
+        boolean b = masterTaskExecThread.existsValidWorkerGroup("test1");
-        masterTaskExecThread.existsValidWorkerGroup("test1");
+        Assert.assertTrue(b);
     }
 
     @Test
@@ -83,17 +93,15 @@ public class MasterTaskExecThreadTest {
 
 
         ProcessService processService = Mockito.mock(ProcessService.class);
-        ApplicationContext applicationContext = Mockito.mock(ApplicationContext.class);
+        Mockito.when(this.springApplicationContext.getBean(ProcessService.class))
-        SpringApplicationContext springApplicationContext = new SpringApplicationContext();
+                .thenReturn(processService);
-        springApplicationContext.setApplicationContext(applicationContext);
-        Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService);
 
         TaskInstance taskInstance = getTaskInstance();
         Mockito.when(processService.findTaskInstanceById(252612))
                 .thenReturn(taskInstance);
 
         Mockito.when(processService.updateTaskInstance(taskInstance))
-        .thenReturn(true);
+                .thenReturn(true);
 
         MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(taskInstance);
         masterTaskExecThread.pauseTask();