diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/dolphin-daemon.j2 b/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/dolphin-daemon.j2 index ab99ffda47..1dc4bac0ab 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/dolphin-daemon.j2 +++ b/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/dolphin-daemon.j2 @@ -39,7 +39,7 @@ export HOSTNAME=`hostname` DOLPHINSCHEDULER_LIB_JARS={{dolphin_lib_jars}} -DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms4g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" +DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms1g -Xss512k -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=10m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" STOP_TIMEOUT=5 log={{dolphin_log_dir}}/dolphinscheduler-$command-$HOSTNAME.out diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/dolphin-daemon.sh.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/dolphin-daemon.sh.j2 index 0802b74750..c5cc11fb62 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/dolphin-daemon.sh.j2 +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/dolphin-daemon.sh.j2 @@ -39,7 +39,7 @@ export HOSTNAME=`hostname` DOLPHINSCHEDULER_LIB_JARS={{dolphin_lib_jars}} -DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms4g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" +DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms1g -Xss512k -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=10m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" STOP_TIMEOUT=5 log={{dolphin_log_dir}}/dolphinscheduler-$command-$HOSTNAME.out diff --git a/docker/build/conf/nginx/dolphinscheduler.conf b/docker/build/conf/nginx/dolphinscheduler.conf index 9c2c3913dc..a594385a0e 100644 --- a/docker/build/conf/nginx/dolphinscheduler.conf +++ b/docker/build/conf/nginx/dolphinscheduler.conf @@ -24,6 +24,9 @@ server { root /opt/dolphinscheduler/ui; index index.html index.html; } + location /dolphinscheduler/ui{ + alias /opt/dolphinscheduler/ui; + } location /dolphinscheduler { proxy_pass http://FRONTEND_API_SERVER_HOST:FRONTEND_API_SERVER_PORT; proxy_set_header Host $host; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java index e4817ddc18..3f53c535d2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java @@ -22,7 +22,6 @@ import org.springframework.boot.web.servlet.ServletComponentScan; import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.FilterType; -import springfox.documentation.swagger2.annotations.EnableSwagger2; @SpringBootApplication @ServletComponentScan diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java index 10dc8e4ce5..7e9473d81c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.User; import io.swagger.annotations.*; import org.slf4j.Logger; @@ -204,6 +205,39 @@ public class ProcessInstanceController extends BaseController { return returnDataList(result); } + /** + * query top n process instance order by running duration + * + * @param loginUser login user + * @param projectName project name + * @param size number of process instance + * @param startTime start time + * @param endTime end time + * @return list of process instance + */ + @ApiOperation(value = "queryTopNLongestRunningProcessInstance", notes = "QUERY_TOPN_LONGEST_RUNNING_PROCESS_INSTANCE_NOTES") + @ApiImplicitParams({ + @ApiImplicitParam(name = "size", value = "PROCESS_INSTANCE_SIZE", dataType = "Int", example = "10"), + @ApiImplicitParam(name = "startTime", value = "PROCESS_INSTANCE_START_TIME", dataType = "String"), + @ApiImplicitParam(name = "endTime", value = "PROCESS_INSTANCE_END_TIME", dataType = "String"), + }) + @GetMapping(value = "/top-n") + @ResponseStatus(HttpStatus.OK) + @ApiException(QUERY_PROCESS_INSTANCE_BY_ID_ERROR) + public Result queryTopNLongestRunningProcessInstance(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, + @RequestParam("size") Integer size, + @RequestParam(value = "startTime",required = true) String startTime, + @RequestParam(value = "endTime",required = true) String endTime + + ) { + projectName=ParameterUtils.handleEscapes(projectName); + logger.info("query top {} SUCCESS process instance order by running time whprojectNameich started between {} and {} ,login user:{},project name:{}", size, startTime, endTime, + loginUser.getUserName(), projectName); + Map result=processInstanceService.queryTopNLongestRunningProcessInstance(loginUser, projectName, size, startTime, endTime); + return returnDataList(result); + } + /** * delete process instance by id, at the same time, * delete task instance and their mapping relation data @@ -220,9 +254,9 @@ public class ProcessInstanceController extends BaseController { @GetMapping(value = "/delete") @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_PROCESS_INSTANCE_BY_ID_ERROR) - public Result deleteProcessInstanceById(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, - @RequestParam("processInstanceId") Integer processInstanceId + public Result deleteProcessInstanceById(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, + @RequestParam("processInstanceId") Integer processInstanceId ) { logger.info("delete process instance by id, login user:{}, project name:{}, process instance id:{}", loginUser.getUserName(), projectName, processInstanceId); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java index a9a5514027..f55492a69e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java @@ -52,7 +52,7 @@ public class SchedulerController extends BaseController { public static final String DEFAULT_WARNING_TYPE = "NONE"; public static final String DEFAULT_NOTIFY_GROUP_ID = "1"; public static final String DEFAULT_FAILURE_POLICY = "CONTINUE"; - + public static final String DEFAULT_PROCESS_INSTANCE_PRIORITY = "MEDIUM"; @Autowired private SchedulerService schedulerService; @@ -99,7 +99,7 @@ public class SchedulerController extends BaseController { @RequestParam(value = "receivers", required = false) String receivers, @RequestParam(value = "receiversCc", required = false) String receiversCc, @RequestParam(value = "workerGroup", required = false, defaultValue = "default") String workerGroup, - @RequestParam(value = "processInstancePriority", required = false) Priority processInstancePriority) throws IOException { + @RequestParam(value = "processInstancePriority", required = false, defaultValue = DEFAULT_PROCESS_INSTANCE_PRIORITY) Priority processInstancePriority) throws IOException { logger.info("login user {}, project name: {}, process name: {}, create schedule: {}, warning type: {}, warning group id: {}," + "failure policy: {},receivers : {},receiversCc : {},processInstancePriority : {}, workGroupId:{}", loginUser.getUserName(), projectName, processDefinitionId, schedule, warningType, warningGroupId, diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageController.java new file mode 100644 index 0000000000..895d2cd8da --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageController.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.controller; + +import org.apache.dolphinscheduler.api.service.WorkFlowLineageService; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import io.swagger.annotations.ApiParam; +import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.*; +import springfox.documentation.annotations.ApiIgnore; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_WORKFLOW_LINEAGE_ERROR; + +@RestController +@RequestMapping("lineages/{projectId}") +public class WorkFlowLineageController extends BaseController { + private static final Logger logger = LoggerFactory.getLogger(WorkFlowLineageController.class); + + @Autowired + private WorkFlowLineageService workFlowLineageService; + + @GetMapping(value="/list-name") + @ResponseStatus(HttpStatus.OK) + public Result> queryWorkFlowLineageByName(@ApiIgnore @RequestParam(value = "searchVal", required = false) String searchVal, @ApiParam(name = "projectId", value = "PROJECT_ID", required = true) @PathVariable int projectId) { + try { + searchVal = ParameterUtils.handleEscapes(searchVal); + Map result = workFlowLineageService.queryWorkFlowLineageByName(searchVal,projectId); + return returnDataList(result); + } catch (Exception e){ + logger.error(QUERY_WORKFLOW_LINEAGE_ERROR.getMsg(),e); + return error(QUERY_WORKFLOW_LINEAGE_ERROR.getCode(), QUERY_WORKFLOW_LINEAGE_ERROR.getMsg()); + } + } + + @GetMapping(value="/list-ids") + @ResponseStatus(HttpStatus.OK) + public Result> queryWorkFlowLineageByIds(@ApiIgnore @RequestParam(value = "ids", required = false) String ids,@ApiParam(name = "projectId", value = "PROJECT_ID", required = true) @PathVariable int projectId) { + + try { + ids = ParameterUtils.handleEscapes(ids); + Set idsSet = new HashSet<>(); + if(ids != null) { + String[] idsStr = ids.split(","); + for (String id : idsStr) + { + idsSet.add(Integer.parseInt(id)); + } + } + + Map result = workFlowLineageService.queryWorkFlowLineageByIds(idsSet, projectId); + return returnDataList(result); + } catch (Exception e){ + logger.error(QUERY_WORKFLOW_LINEAGE_ERROR.getMsg(),e); + return error(QUERY_WORKFLOW_LINEAGE_ERROR.getCode(), QUERY_WORKFLOW_LINEAGE_ERROR.getMsg()); + } + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java index 6b0391f111..fa7588f2ed 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java @@ -62,7 +62,7 @@ public class TaskCountDto { case SUBMITTED_SUCCESS: submittedSuccess += taskInstanceStateCount.getCount(); break; - case RUNNING_EXEUTION: + case RUNNING_EXECUTION: runningExeution += taskInstanceStateCount.getCount(); break; case READY_PAUSE: @@ -99,7 +99,7 @@ public class TaskCountDto { } this.taskCountDtos = new ArrayList<>(); this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.SUBMITTED_SUCCESS, submittedSuccess)); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.RUNNING_EXEUTION, runningExeution)); + this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.RUNNING_EXECUTION, runningExeution)); this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.READY_PAUSE, readyPause)); this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.PAUSE, pause)); this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.READY_STOP, readyStop)); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java index ec0b8f0a76..8e90b4cb08 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -168,15 +168,15 @@ public enum Status { PREVIEW_SCHEDULE_ERROR(10139,"preview schedule error", "预览调度配置错误"), PARSE_TO_CRON_EXPRESSION_ERROR(10140,"parse cron to cron expression error", "解析调度表达式错误"), SCHEDULE_START_TIME_END_TIME_SAME(10141,"The start time must not be the same as the end", "开始时间不能和结束时间一样"), - DELETE_TENANT_BY_ID_FAIL(10142,"delete tenant by id fail, for there are {0} process instances in executing using it", "删除租户失败，有[{0}]个运行中的工作流实例正在使用"), - DELETE_TENANT_BY_ID_FAIL_DEFINES(10143,"delete tenant by id fail, for there are {0} process definitions using it", "删除租户失败，有[{0}]个工作流定义正在使用"), - DELETE_TENANT_BY_ID_FAIL_USERS(10144,"delete tenant by id fail, for there are {0} users using it", "删除租户失败，有[{0}]个用户正在使用"), - DELETE_WORKER_GROUP_BY_ID_FAIL(10145,"delete worker group by id fail, for there are {0} process instances in executing using it", "删除Worker分组失败，有[{0}]个运行中的工作流实例正在使用"), - QUERY_WORKER_GROUP_FAIL(10146,"query worker group fail ", "查询worker分组失败"), - DELETE_WORKER_GROUP_FAIL(10147,"delete worker group fail ", "删除worker分组失败"), + DELETE_TENANT_BY_ID_FAIL(100142,"delete tenant by id fail, for there are {0} process instances in executing using it", "删除租户失败，有[{0}]个运行中的工作流实例正在使用"), + DELETE_TENANT_BY_ID_FAIL_DEFINES(100143,"delete tenant by id fail, for there are {0} process definitions using it", "删除租户失败，有[{0}]个工作流定义正在使用"), + DELETE_TENANT_BY_ID_FAIL_USERS(100144,"delete tenant by id fail, for there are {0} users using it", "删除租户失败，有[{0}]个用户正在使用"), + DELETE_WORKER_GROUP_BY_ID_FAIL(100145,"delete worker group by id fail, for there are {0} process instances in executing using it", "删除Worker分组失败，有[{0}]个运行中的工作流实例正在使用"), + QUERY_WORKER_GROUP_FAIL(100146,"query worker group fail ", "查询worker分组失败"), + DELETE_WORKER_GROUP_FAIL(100147,"delete worker group fail ", "删除worker分组失败"), + QUERY_WORKFLOW_LINEAGE_ERROR(10143,"query workflow lineage error", "查询血缘失败"), COPY_PROCESS_DEFINITION_ERROR(10148,"copy process definition error", "复制工作流错误"), USER_DISABLED(10149,"The current user is disabled", "当前用户已停用"), - UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"), UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"), RESOURCE_NOT_EXIST(20004, "resource not exist", "资源不存在"), @@ -249,7 +249,8 @@ public enum Status { COMMAND_STATE_COUNT_ERROR(80001,"task instance state count error", "查询各状态任务实例数错误"), - + NEGTIVE_SIZE_NUMBER_ERROR(80002,"query size number error","查询size错误"), + START_TIME_BIGGER_THAN_END_TIME_ERROR(80003,"start time bigger than end time error","开始时间在结束时间之后错误"), QUEUE_COUNT_ERROR(90001,"queue count error", "查询队列数据错误"), KERBEROS_STARTUP_STATE(100001,"get kerberos startup state error", "获取kerberos启动状态错误"), diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java index 001a10d08a..f3dcbfa237 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java @@ -177,7 +177,7 @@ public class AlertGroupService extends BaseService{ * @param id alert group id * @return delete result code */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public Map delAlertgroupById(User loginUser, int id) { Map result = new HashMap<>(5); result.put(Constants.STATUS, false); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index caedf854e7..41374f4478 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -527,7 +527,7 @@ public class DataSourceService extends BaseService{ parameterMap.put(Constants.DATABASE, database); parameterMap.put(Constants.JDBC_URL, jdbcUrl); parameterMap.put(Constants.USER, userName); - parameterMap.put(Constants.PASSWORD, password); + parameterMap.put(Constants.PASSWORD, CommonUtils.encodePassword(password)); if (CommonUtils.getKerberosStartupState() && (type == DbType.HIVE || type == DbType.SPARK)){ parameterMap.put(Constants.PRINCIPAL,principal); @@ -600,7 +600,7 @@ public class DataSourceService extends BaseService{ * @param datasourceId data source id * @return delete result code */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public Result delete(User loginUser, int datasourceId) { Result result = new Result(); try { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java index 2f44dee304..3c7b421d5e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java @@ -16,6 +16,9 @@ */ package org.apache.dolphinscheduler.api.service; +import java.nio.charset.StandardCharsets; +import javax.annotation.PreDestroy; +import org.apache.commons.lang.ArrayUtils; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; @@ -29,8 +32,6 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import javax.annotation.PreDestroy; - /** * log service */ @@ -39,17 +40,19 @@ public class LoggerService { private static final Logger logger = LoggerFactory.getLogger(LoggerService.class); + private static final String LOG_HEAD_FORMAT = "[LOG-PATH]: %s, [HOST]: %s%s"; + @Autowired private ProcessService processService; private final LogClientService logClient; - public LoggerService(){ + public LoggerService() { logClient = new LogClientService(); } @PreDestroy - public void close(){ + public void close() { logClient.close(); } @@ -65,24 +68,34 @@ public class LoggerService { TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); - if (taskInstance == null || StringUtils.isBlank(taskInstance.getHost())){ - return new Result(Status.TASK_INSTANCE_NOT_FOUND.getCode(), Status.TASK_INSTANCE_NOT_FOUND.getMsg()); + if (taskInstance == null || StringUtils.isBlank(taskInstance.getHost())) { + return Result.error(Status.TASK_INSTANCE_NOT_FOUND); } String host = getHost(taskInstance.getHost()); Result result = new Result(Status.SUCCESS.getCode(), Status.SUCCESS.getMsg()); - logger.info("log host : {} , logPath : {} , logServer port : {}",host,taskInstance.getLogPath(),Constants.RPC_PORT); + logger.info("log host : {} , logPath : {} , logServer port : {}", host, taskInstance.getLogPath(), + Constants.RPC_PORT); + + StringBuilder log = new StringBuilder(); + if (skipLineNum == 0) { + String head = String.format(LOG_HEAD_FORMAT, + taskInstance.getLogPath(), + host, + Constants.SYSTEM_LINE_SEPARATOR); + log.append(head); + } + + log.append(logClient + .rollViewLog(host, Constants.RPC_PORT, taskInstance.getLogPath(), skipLineNum, limit)); - String log = logClient.rollViewLog(host, Constants.RPC_PORT, taskInstance.getLogPath(),skipLineNum,limit); result.setData(log); return result; } - - /** * get log size * @@ -91,22 +104,27 @@ public class LoggerService { */ public byte[] getLogBytes(int taskInstId) { TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); - if (taskInstance == null || StringUtils.isBlank(taskInstance.getHost())){ + if (taskInstance == null || StringUtils.isBlank(taskInstance.getHost())) { throw new RuntimeException("task instance is null or host is null"); } String host = getHost(taskInstance.getHost()); - - return logClient.getLogBytes(host, Constants.RPC_PORT, taskInstance.getLogPath()); + byte[] head = String.format(LOG_HEAD_FORMAT, + taskInstance.getLogPath(), + host, + Constants.SYSTEM_LINE_SEPARATOR).getBytes(StandardCharsets.UTF_8); + return ArrayUtils.addAll(head, + logClient.getLogBytes(host, Constants.RPC_PORT, taskInstance.getLogPath())); } /** * get host + * * @param address address * @return old version return true ,otherwise return false */ - private String getHost(String address){ - if (Host.isOldVersion(address)){ + private String getHost(String address) { + if (Host.isOldVersion(address)) { return address; } return Host.of(address).getIp(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index e23ed0002a..b3d56e5982 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -438,7 +438,7 @@ public class ProcessDefinitionService extends BaseDAGService { * @param processDefinitionId process definition id * @return delete result code */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public Map deleteProcessDefinitionById(User loginUser, String projectName, Integer processDefinitionId) { Map result = new HashMap<>(5); @@ -504,7 +504,7 @@ public class ProcessDefinitionService extends BaseDAGService { * @param releaseState release state * @return release result code */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public Map releaseProcessDefinition(User loginUser, String projectName, int id, int releaseState) { HashMap result = new HashMap<>(); Project project = projectMapper.queryByName(projectName); @@ -751,7 +751,7 @@ public class ProcessDefinitionService extends BaseDAGService { * @param currentProjectName current project name * @return import process */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public Map importProcessDefinition(User loginUser, MultipartFile file, String currentProjectName) { Map result = new HashMap<>(5); String processMetaJson = FileUtils.file2String(file); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java index e3d6311d46..e4a00f3895 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java @@ -98,6 +98,53 @@ public class ProcessInstanceService extends BaseDAGService { @Autowired UsersService usersService; + /** + * return top n SUCCESS process instance order by running time which started between startTime and endTime + * @param loginUser + * @param projectName + * @param size + * @param startTime + * @param endTime + * @return + */ + public Map queryTopNLongestRunningProcessInstance(User loginUser, String projectName, int size, String startTime, String endTime) { + Map result = new HashMap<>(); + + Project project = projectMapper.queryByName(projectName); + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultEnum = (Status) checkResult.get(Constants.STATUS); + if (resultEnum != Status.SUCCESS) { + return checkResult; + } + + if (0 > size) { + putMsg(result, Status.NEGTIVE_SIZE_NUMBER_ERROR, size); + return result; + } + if (Objects.isNull(startTime)) { + putMsg(result, Status.DATA_IS_NULL, Constants.START_TIME); + return result; + } + Date start = DateUtils.stringToDate(startTime); + if (Objects.isNull(endTime)) { + putMsg(result, Status.DATA_IS_NULL, Constants.END_TIME); + return result; + } + Date end = DateUtils.stringToDate(endTime); + if(start == null || end == null) { + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "startDate,endDate"); + return result; + } + if (start.getTime() > end.getTime()) { + putMsg(result, Status.START_TIME_BIGGER_THAN_END_TIME_ERROR, startTime, endTime); + return result; + } + + List processInstances = processInstanceMapper.queryTopNProcessInstance(size, start, end, ExecutionStatus.SUCCESS); + result.put(DATA_LIST, processInstances); + putMsg(result, Status.SUCCESS); + return result; + } /** * query process instance by id * @@ -466,7 +513,7 @@ public class ProcessInstanceService extends BaseDAGService { * @param processInstanceId process instance id * @return delete result code */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public Map deleteProcessInstanceById(User loginUser, String projectName, Integer processInstanceId) { Map result = new HashMap<>(5); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index ded6594192..3cb715b964 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -86,7 +86,7 @@ public class ResourcesService extends BaseService { * @param currentDir current directory * @return create directory result */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public Result createDirectory(User loginUser, String name, String description, @@ -160,7 +160,7 @@ public class ResourcesService extends BaseService { * @param currentDir current directory * @return create result code */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public Result createResource(User loginUser, String name, String desc, @@ -288,7 +288,7 @@ public class ResourcesService extends BaseService { * @param type resource type * @return update result code */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public Result updateResource(User loginUser, int resourceId, String name, @@ -827,7 +827,7 @@ public class ResourcesService extends BaseService { * @param content content * @return create result code */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public Result onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix, String desc, String content,int pid,String currentDirectory) { Result result = new Result(); // if resource upload startup @@ -889,7 +889,7 @@ public class ResourcesService extends BaseService { * @param content content * @return update result cod */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public Result updateResourceContent(int resourceId, String content) { Result result = new Result(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java index ed856d5094..78a36c639a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java @@ -47,7 +47,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import java.io.IOException; import java.text.ParseException; import java.util.*; @@ -95,9 +94,8 @@ public class SchedulerService extends BaseService { * @param receiversCc receivers cc * @param workerGroup worker group * @return create result code - * @throws IOException ioexception */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public Map insertSchedule(User loginUser, String projectName, Integer processDefineId, String schedule, @@ -107,7 +105,7 @@ public class SchedulerService extends BaseService { String receivers, String receiversCc, Priority processInstancePriority, - String workerGroup) throws IOException { + String workerGroup) { Map result = new HashMap(5); @@ -192,9 +190,8 @@ public class SchedulerService extends BaseService { * @param receivers receivers * @param scheduleStatus schedule status * @return update result code - * @throws IOException ioexception */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public Map updateSchedule(User loginUser, String projectName, Integer id, @@ -206,7 +203,7 @@ public class SchedulerService extends BaseService { String receiversCc, ReleaseState scheduleStatus, Priority processInstancePriority, - String workerGroup) throws IOException { + String workerGroup) { Map result = new HashMap(5); Project project = projectMapper.queryByName(projectName); @@ -296,7 +293,7 @@ public class SchedulerService extends BaseService { * @param scheduleStatus schedule status * @return publish result code */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public Map setScheduleState(User loginUser, String projectName, Integer id, diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SessionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SessionService.java index 2588dd0e65..b4aab962ef 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SessionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SessionService.java @@ -81,7 +81,7 @@ public class SessionService extends BaseService{ * @param ip ip * @return session string */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public String createSession(User user, String ip) { Session session = null; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java index 8a0bf748bb..bb92c9ccf7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java @@ -300,7 +300,7 @@ public class UdfFuncService extends BaseService{ * @param id udf function id * @return delete result code */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public Result delete(int id) { Result result = new Result(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java index bf51261e15..cbd795cce4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java @@ -138,14 +138,14 @@ public class UsersService extends BaseService { } - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public User createUser(String userName, String userPassword, String email, int tenantId, String phone, String queue, - int state) throws Exception { + int state) { User user = new User(); Date now = new Date(); @@ -430,7 +430,7 @@ public class UsersService extends BaseService { * @param projectIds project id array * @return grant result code */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public Map grantProject(User loginUser, int userId, String projectIds) { Map result = new HashMap<>(5); result.put(Constants.STATUS, false); @@ -480,7 +480,7 @@ public class UsersService extends BaseService { * @param resourceIds resource id array * @return grant result code */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public Map grantResources(User loginUser, int userId, String resourceIds) { Map result = new HashMap<>(5); //only admin can operate @@ -577,7 +577,7 @@ public class UsersService extends BaseService { * @param udfIds udf id array * @return grant result code */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public Map grantUDFFunction(User loginUser, int userId, String udfIds) { Map result = new HashMap<>(5); @@ -624,7 +624,7 @@ public class UsersService extends BaseService { * @param datasourceIds data source id array * @return grant result code */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public Map grantDataSource(User loginUser, int userId, String datasourceIds) { Map result = new HashMap<>(5); result.put(Constants.STATUS, false); @@ -917,10 +917,9 @@ public class UsersService extends BaseService { * @param repeatPassword repeat password * @param email email * @return register result code - * @throws Exception exception */ - @Transactional(rollbackFor = Exception.class) - public Map registerUser(String userName, String userPassword, String repeatPassword, String email) throws Exception { + @Transactional(rollbackFor = RuntimeException.class) + public Map registerUser(String userName, String userPassword, String repeatPassword, String email) { Map result = new HashMap<>(5); //check user params diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkFlowLineageService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkFlowLineageService.java new file mode 100644 index 0000000000..29fd99c0f7 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkFlowLineageService.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper; +import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage; +import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.*; + +@Service +public class WorkFlowLineageService extends BaseService { + + @Autowired + private WorkFlowLineageMapper workFlowLineageMapper; + + public Map queryWorkFlowLineageByName(String workFlowName, int projectId) { + Map result = new HashMap<>(5); + List workFlowLineageList = workFlowLineageMapper.queryByName(workFlowName, projectId); + result.put(Constants.DATA_LIST, workFlowLineageList); + putMsg(result, Status.SUCCESS); + return result; + } + + private List getWorkFlowRelationRecursion(Set ids, List workFlowRelations,Set sourceIds) { + for(int id : ids) { + sourceIds.addAll(ids); + List workFlowRelationsTmp = workFlowLineageMapper.querySourceTarget(id); + if(workFlowRelationsTmp != null && !workFlowRelationsTmp.isEmpty()) { + Set idsTmp = new HashSet<>(); + for(WorkFlowRelation workFlowRelation:workFlowRelationsTmp) { + if(!sourceIds.contains(workFlowRelation.getTargetWorkFlowId())){ + idsTmp.add(workFlowRelation.getTargetWorkFlowId()); + } + } + workFlowRelations.addAll(workFlowRelationsTmp); + getWorkFlowRelationRecursion(idsTmp, workFlowRelations,sourceIds); + } + } + return workFlowRelations; + } + + public Map queryWorkFlowLineageByIds(Set ids,int projectId) { + Map result = new HashMap<>(5); + List workFlowLineageList = workFlowLineageMapper.queryByIds(ids, projectId); + Map workFlowLists = new HashMap<>(5); + Set idsV = new HashSet<>(); + if(ids == null || ids.isEmpty()){ + for(WorkFlowLineage workFlowLineage:workFlowLineageList) { + idsV.add(workFlowLineage.getWorkFlowId()); + } + } else { + idsV = ids; + } + List workFlowRelations = new ArrayList<>(); + Set sourceIds = new HashSet<>(); + getWorkFlowRelationRecursion(idsV, workFlowRelations, sourceIds); + + Set idSet = new HashSet<>(); + //If the incoming parameter is not empty, you need to add downstream workflow detail attributes + if(ids != null && !ids.isEmpty()) { + for(WorkFlowRelation workFlowRelation : workFlowRelations) { + idSet.add(workFlowRelation.getTargetWorkFlowId()); + } + for(int id : ids){ + idSet.remove(id); + } + if(!idSet.isEmpty()) { + workFlowLineageList.addAll(workFlowLineageMapper.queryByIds(idSet, projectId)); + } + } + + workFlowLists.put("workFlowList",workFlowLineageList); + workFlowLists.put("workFlowRelationList",workFlowRelations); + result.put(Constants.DATA_LIST, workFlowLists); + putMsg(result, Status.SUCCESS); + return result; + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java index 55bb3d336c..ea458c9764 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java @@ -149,6 +149,7 @@ public class ZooKeeperState { return sendThread.ret; } catch (InterruptedException e) { logger.error("send " + cmd + " to server " + host + ":" + port + " failed!", e); + Thread.currentThread().interrupt(); } return ""; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitor.java index 7ba704916c..be4754544f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitor.java @@ -81,8 +81,7 @@ public class ZookeeperMonitor extends AbstractZKClient { if(ok){ state.getZookeeperInfo(); } - - String hostName = zookeeperServer; + int connections = state.getConnections(); int watches = state.getWatches(); long sent = state.getSent(); @@ -95,7 +94,7 @@ public class ZookeeperMonitor extends AbstractZKClient { int status = ok ? 1 : 0; Date date = new Date(); - ZookeeperRecord zookeeperRecord = new ZookeeperRecord(hostName,connections,watches,sent,received,mode,minLatency,avgLatency,maxLatency,nodeCount,status,date); + ZookeeperRecord zookeeperRecord = new ZookeeperRecord(zookeeperServer,connections,watches,sent,received,mode,minLatency,avgLatency,maxLatency,nodeCount,status,date); list.add(zookeeperRecord); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java index 96fbe4cec1..5189097e68 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java @@ -74,7 +74,8 @@ public class ProcessInstanceControllerTest extends AbstractControllerTest { .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.getCode().intValue()); + assert result != null; + Assert.assertEquals(Status.PROJECT_NOT_FOUNT.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageControllerTest.java new file mode 100644 index 0000000000..7dea15d537 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageControllerTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.controller; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +public class WorkFlowLineageControllerTest extends AbstractControllerTest { + private static Logger logger = LoggerFactory.getLogger(WorkFlowLineageControllerTest.class); + + @Test + public void testQueryWorkFlowLineageByName() throws Exception { + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("searchVal","test"); + MvcResult mvcResult = mockMvc.perform(get("/lineages/1/list-name") + .header("sessionId", sessionId) + .params(paramsMap)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) + .andReturn(); + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); + logger.info(mvcResult.getResponse().getContentAsString()); + + } + + @Test + public void testQueryWorkFlowLineageByIds() throws Exception { + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("ids","1"); + MvcResult mvcResult = mockMvc.perform(get("/lineages/1/list-ids") + .header("sessionId", sessionId) + .params(paramsMap)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) + .andReturn(); + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); + logger.info(mvcResult.getResponse().getContentAsString()); + } + +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java index 14612fcef8..6a9e78600b 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java @@ -180,7 +180,7 @@ public class DataAnalysisServiceTest { List taskInstanceStateCounts = new ArrayList<>(1); ExecuteStatusCount executeStatusCount = new ExecuteStatusCount(); - executeStatusCount.setExecutionStatus(ExecutionStatus.RUNNING_EXEUTION); + executeStatusCount.setExecutionStatus(ExecutionStatus.RUNNING_EXECUTION); taskInstanceStateCounts.add(executeStatusCount); return taskInstanceStateCounts; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java index 00e5275e80..c185868cde 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbConnectType; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; @@ -103,7 +104,28 @@ public class DataSourceServiceTest { public void buildParameter(){ String param = dataSourceService.buildParameter("","", DbType.ORACLE, "192.168.9.1","1521","im" ,"","test","test", DbConnectType.ORACLE_SERVICE_NAME,""); - String expected = "{\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}"; + String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}"; Assert.assertEquals(expected, param); } + + @Test + public void buildParameterWithDecodePassword(){ + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true"); + String param = dataSourceService.buildParameter("name","desc", DbType.MYSQL, "192.168.9.1","1521","im" + ,"","test","123456", null,""); + String expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}"; + Assert.assertEquals(expected, param); + + + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false"); + param = dataSourceService.buildParameter("name","desc", DbType.MYSQL, "192.168.9.1","1521","im" + ,"","test","123456", null,""); + expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"123456\"}"; + Assert.assertEquals(expected, param); + } + + + + + } \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index d4fa2307ec..8db667e28b 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -474,7 +474,7 @@ public class ProcessDefinitionServiceTest { ProcessInstance processInstance = new ProcessInstance(); processInstance.setId(1); processInstance.setName("test_instance"); - processInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + processInstance.setState(ExecutionStatus.RUNNING_EXECUTION); processInstance.setHost("192.168.xx.xx"); processInstance.setStartTime(new Date()); processInstance.setEndTime(new Date()); @@ -486,7 +486,7 @@ public class ProcessDefinitionServiceTest { taskInstance.setTaskType("SHELL"); taskInstance.setId(1); taskInstance.setName("test_task_instance"); - taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION); taskInstance.setHost("192.168.xx.xx"); //task instance not exist diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java index b35614335c..651964bb16 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java @@ -147,6 +147,40 @@ public class ProcessInstanceServiceTest { } + @Test + public void testQueryTopNLongestRunningProcessInstance() { + String projectName = "project_test1"; + User loginUser = getAdminUser(); + Map result = new HashMap<>(5); + putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); + int size=10; + String startTime="2020-01-01 00:00:00"; + String endTime="2020-08-02 00:00:00"; + Date start = DateUtils.getScheduleDate(startTime); + Date end = DateUtils.getScheduleDate(endTime); + + //project auth fail + when(projectMapper.queryByName(projectName)).thenReturn(null); + when(projectService.checkProjectAndAuth(loginUser, null, projectName)).thenReturn(result); + Map proejctAuthFailRes = processInstanceService.queryTopNLongestRunningProcessInstance(loginUser,projectName,size,startTime,endTime); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT, proejctAuthFailRes.get(Constants.STATUS)); + + //project auth success + putMsg(result, Status.SUCCESS, projectName); + Project project = getProject(projectName); + ProcessInstance processInstance = getProcessInstance(); + List processInstanceList = new ArrayList<>(); + processInstanceList.add(processInstance); + when(projectMapper.queryByName(projectName)).thenReturn(project); + when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); + when(usersService.queryUser(loginUser.getId())).thenReturn(loginUser); + when(usersService.getUserIdByName(loginUser.getUserName())).thenReturn(loginUser.getId()); + when(usersService.queryUser(processInstance.getExecutorId())).thenReturn(loginUser); + Map successRes = processInstanceService.queryTopNLongestRunningProcessInstance(loginUser,projectName,size,startTime,endTime); + + Assert.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); + } + @Test public void testQueryProcessInstanceById() { String projectName = "project_test1"; @@ -307,7 +341,7 @@ public class ProcessInstanceServiceTest { //process instance not finish when(processService.findProcessInstanceDetailById(1)).thenReturn(processInstance); - processInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + processInstance.setState(ExecutionStatus.RUNNING_EXECUTION); Map processInstanceNotFinishRes = processInstanceService.updateProcessInstance(loginUser, projectName, 1, shellJson, "2020-02-21 00:00:00", true, Flag.YES, "", ""); Assert.assertEquals(Status.PROCESS_INSTANCE_STATE_OPERATION_ERROR, processInstanceNotFinishRes.get(Constants.STATUS)); @@ -414,7 +448,7 @@ public class ProcessInstanceServiceTest { ProcessInstance processInstance = getProcessInstance(); processInstance.setProcessInstanceJson(shellJson); TaskInstance taskInstance = getTaskInstance(); - taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION); taskInstance.setStartTime(new Date()); when(processInstanceMapper.queryDetailById(1)).thenReturn(processInstance); when(taskInstanceMapper.queryByInstanceIdAndName(Mockito.anyInt(), Mockito.any())).thenReturn(taskInstance); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkFlowLineageServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkFlowLineageServiceTest.java new file mode 100644 index 0000000000..999e079bf5 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkFlowLineageServiceTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.EncryptionUtils; +import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage; +import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation; +import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.*; + +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class WorkFlowLineageServiceTest { + + @InjectMocks + private WorkFlowLineageService workFlowLineageService; + + @Mock + private WorkFlowLineageMapper workFlowLineageMapper; + + @Test + public void testQueryWorkFlowLineageByName() { + String searchVal = "test"; + when(workFlowLineageMapper.queryByName(searchVal, 1)).thenReturn(getWorkFlowLineages()); + Map result = workFlowLineageService.queryWorkFlowLineageByName(searchVal,1); + List workFlowLineageList = (List)result.get(Constants.DATA_LIST); + Assert.assertTrue(workFlowLineageList.size()>0); + } + + @Test + public void testQueryWorkFlowLineageByIds() { + + Set ids = new HashSet<>(); + ids.add(1); + ids.add(2); + + when(workFlowLineageMapper.queryByIds(ids, 1)).thenReturn(getWorkFlowLineages()); + when(workFlowLineageMapper.querySourceTarget(1)).thenReturn(getWorkFlowRelation()); + Map result = workFlowLineageService.queryWorkFlowLineageByIds(ids,1); + Map workFlowLists = (Map)result.get(Constants.DATA_LIST); + List workFlowLineages = (List)workFlowLists.get("workFlowList"); + List workFlowRelations = (List)workFlowLists.get("workFlowRelationList"); + Assert.assertTrue(workFlowLineages.size()>0); + Assert.assertTrue(workFlowRelations.size()>0); + } + + private List getWorkFlowLineages() { + List workFlowLineages = new ArrayList<>(); + WorkFlowLineage workFlowLineage = new WorkFlowLineage(); + workFlowLineage.setWorkFlowId(1); + workFlowLineage.setWorkFlowName("testdag"); + workFlowLineages.add(workFlowLineage); + return workFlowLineages; + } + + private List getWorkFlowRelation(){ + List workFlowRelations = new ArrayList<>(); + WorkFlowRelation workFlowRelation = new WorkFlowRelation(); + workFlowRelation.setSourceWorkFlowId(1); + workFlowRelation.setTargetWorkFlowId(2); + workFlowRelations.add(workFlowRelation); + return workFlowRelations; + } + +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 57491c96b7..4cb09a1a56 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -831,7 +831,7 @@ public final class Constants { public static final int[] NOT_TERMINATED_STATES = new int[]{ ExecutionStatus.SUBMITTED_SUCCESS.ordinal(), - ExecutionStatus.RUNNING_EXEUTION.ordinal(), + ExecutionStatus.RUNNING_EXECUTION.ordinal(), ExecutionStatus.READY_PAUSE.ordinal(), ExecutionStatus.READY_STOP.ordinal(), ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal(), @@ -978,10 +978,23 @@ public final class Constants { public static final int NORAML_NODE_STATUS = 0; public static final int ABNORMAL_NODE_STATUS = 1; + public static final String START_TIME = "start time"; + public static final String END_TIME = "end time"; + /** + * system line separator + */ + public static final String SYSTEM_LINE_SEPARATOR = System.getProperty("line.separator"); + /** * net system properties */ public static final String DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE = "dolphin.scheduler.network.interface.preferred"; + /** + * datasource encryption salt + */ + public static final String DATASOURCE_ENCRYPTION_SALT_DEFAULT = "!@#$%^&*"; + public static final String DATASOURCE_ENCRYPTION_ENABLE = "datasource.encryption.enable"; + public static final String DATASOURCE_ENCRYPTION_SALT = "datasource.encryption.salt"; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java index 7cc073e09e..6ea02ef096 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java @@ -43,7 +43,7 @@ public enum ExecutionStatus { * 11 waiting depend node complete */ SUBMITTED_SUCCESS(0, "submit success"), - RUNNING_EXEUTION(1, "running"), + RUNNING_EXECUTION(1, "running"), READY_PAUSE(2, "ready pause"), PAUSE(3, "pause"), READY_STOP(4, "ready stop"), @@ -126,7 +126,7 @@ public enum ExecutionStatus { * @return status */ public boolean typeIsRunning(){ - return this == RUNNING_EXEUTION || this == WAITTING_DEPEND; + return this == RUNNING_EXECUTION || this == WAITTING_DEPEND; } /** diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskStateType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskStateType.java index 200f90709a..11ab8560b7 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskStateType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskStateType.java @@ -50,7 +50,7 @@ public enum TaskStateType { }; case RUNNING: return new int[]{ExecutionStatus.SUBMITTED_SUCCESS.ordinal(), - ExecutionStatus.RUNNING_EXEUTION.ordinal(), + ExecutionStatus.RUNNING_EXECUTION.ordinal(), ExecutionStatus.READY_PAUSE.ordinal(), ExecutionStatus.READY_STOP.ordinal()}; case WAITTING: diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java index 31e457f105..ae4b94bf5b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java @@ -36,6 +36,7 @@ public enum TaskType { * 10 DATAX * 11 CONDITIONS * 12 SQOOP + * 13 WATERDROP */ SHELL(0, "shell"), SQL(1, "sql"), @@ -49,7 +50,8 @@ public enum TaskType { HTTP(9, "http"), DATAX(10, "datax"), CONDITIONS(11, "conditions"), - SQOOP(12, "sqoop"); + SQOOP(12, "sqoop"), + WATERDROP(13, "waterdrop"); TaskType(int code, String descp){ this.code = code; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java index aafdb86014..9a24fdb407 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java @@ -78,7 +78,7 @@ public abstract class AbstractShell { /** * If or not script finished executing */ - private volatile AtomicBoolean completed; + private AtomicBoolean completed; public AbstractShell() { this(0L); @@ -202,7 +202,7 @@ public abstract class AbstractShell { } catch (InterruptedException ie) { logger.warn("Interrupted while reading the error and in stream", ie); } - completed.set(true); + completed.compareAndSet(false,true); //the timeout thread handling //taken care in finally block if (exitCode != 0 || errMsg.length() > 0) { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java index a9a124547a..7d47ffe738 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java @@ -231,6 +231,8 @@ public class ThreadUtils { public static void sleep(final long millis) { try { Thread.sleep(millis); - } catch (final InterruptedException ignore) {} + } catch (final InterruptedException ignore) { + Thread.currentThread().interrupt(); + } } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java index 0c23d5c2ae..6722c23037 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; +import org.apache.commons.codec.binary.Base64; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.hadoop.conf.Configuration; @@ -23,8 +24,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.net.URL; +import java.nio.charset.StandardCharsets; /** * common utils @@ -32,6 +33,8 @@ import java.net.URL; public class CommonUtils { private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class); + private static final Base64 BASE64 = new Base64(); + private CommonUtils() { throw new IllegalStateException("CommonUtils class"); } @@ -90,4 +93,45 @@ public class CommonUtils { PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)); } } + + /** + * encode password + * @param password + * @return + */ + public static String encodePassword(String password) { + if(StringUtils.isEmpty(password)){return StringUtils.EMPTY; } + //if encryption is not turned on, return directly + boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false); + if ( !encryptionEnable){ return password; } + + // Using Base64 + salt to process password + String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); + String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8))) ; + return new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8))); + } + + /** + * decode password + * @param password + * @return + */ + public static String decodePassword(String password) { + if(StringUtils.isEmpty(password)){return StringUtils.EMPTY ; } + + //if encryption is not turned on, return directly + boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false); + if ( !encryptionEnable){ return password; } + + // Using Base64 + salt to process password + String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); + String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8) ; + if(!passwordWithSalt.startsWith(salt)){ + logger.warn("There is a password and salt mismatch: {} ",password); + return password; + } + return new String(BASE64.decode(passwordWithSalt.substring(salt.length())), StandardCharsets.UTF_8) ; + } + + } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index fce59d6548..cf432a17d5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -448,7 +448,7 @@ public class HadoopUtils implements Closeable { case Constants.RUNNING: default: - return ExecutionStatus.RUNNING_EXEUTION; + return ExecutionStatus.RUNNING_EXECUTION; } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java index 98d9cf16ec..8ea15314a8 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java @@ -18,41 +18,116 @@ package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.Constants; import org.apache.http.HttpEntity; +import org.apache.http.client.config.AuthSchemes; +import org.apache.http.client.config.CookieSpecs; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; import java.io.IOException; - +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.X509Certificate; +import java.util.Arrays; /** * http utils */ public class HttpUtils { - - + + public static final Logger logger = LoggerFactory.getLogger(HttpUtils.class); + private HttpUtils() { + + } + + public static CloseableHttpClient getInstance(){ + return HttpClientInstance.httpClient; + } + + private static class HttpClientInstance{ + private static final CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(requestConfig).build(); + } + + + private static PoolingHttpClientConnectionManager cm; + + private static SSLContext ctx = null; + + private static SSLConnectionSocketFactory socketFactory; + + private static RequestConfig requestConfig; + + private static Registry socketFactoryRegistry; + + private static X509TrustManager xtm = new X509TrustManager() { + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType) { + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType) { + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return null; + } + }; + + static { + try { + ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS); + ctx.init(null, new TrustManager[] { xtm }, null); + } catch (NoSuchAlgorithmException e) { + logger.error("SSLContext init with NoSuchAlgorithmException", e); + } catch (KeyManagementException e) { + logger.error("SSLContext init with KeyManagementException", e); + } + socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE); + /** set timeout、request time、socket timeout */ + requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES) + .setExpectContinueEnabled(Boolean.TRUE) + .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST)) + .setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC)) + .setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT) + .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true) + .build(); + socketFactoryRegistry = RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build(); + cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + cm.setDefaultMaxPerRoute(60); + cm.setMaxTotal(100); + + } + + + /** * get http request content * @param url url * @return http get request response content */ public static String get(String url){ - CloseableHttpClient httpclient = HttpClients.createDefault(); + CloseableHttpClient httpclient = HttpUtils.getInstance(); HttpGet httpget = new HttpGet(url); - /** set timeout、request time、socket timeout */ - RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT) - .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT) - .setSocketTimeout(Constants.SOCKET_TIMEOUT) - .setRedirectsEnabled(true) - .build(); - httpget.setConfig(requestConfig); String responseContent = null; CloseableHttpResponse response = null; @@ -85,12 +160,6 @@ public class HttpUtils { httpget.releaseConnection(); httpget.abort(); } - - try { - httpclient.close(); - } catch (IOException e) { - logger.error(e.getMessage(),e); - } } return responseContent; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java index 2d09f5c439..2d624de1fe 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java @@ -207,7 +207,7 @@ public class ParameterUtils { public static String handleEscapes(String inputString){ if(StringUtils.isNotEmpty(inputString)){ - return inputString.replace("%", "////%"); + return inputString.replace("%", "////%").replaceAll("[\n|\r\t]", "_"); } return inputString; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index 8f0ee327c1..895270766c 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -250,4 +250,14 @@ public class PropertyUtils { } return matchedProperties; } + + /** + * + * @param key + * @param value + */ + public static void setValue(String key, String value) { + properties.setProperty(key,value); + } + } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java index a3492f49fa..84cca93b4c 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java @@ -53,6 +53,8 @@ public class TaskParametersUtils { switch (EnumUtils.getEnum(TaskType.class,taskType)) { case SUB_PROCESS: return JSONUtils.parseObject(parameter, SubProcessParameters.class); + case WATERDROP: + return JSONUtils.parseObject(parameter, ShellParameters.class); case SHELL: return JSONUtils.parseObject(parameter, ShellParameters.class); case PROCEDURE: diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 4a9baae030..a75f964fa2 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -67,4 +67,8 @@ yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s development.state=false # kerberos tgt expire time, unit is hours -kerberos.expire.time=2 \ No newline at end of file +kerberos.expire.time=2 + +# datasource encryption salt +datasource.encryption.enable=false +datasource.encryption.salt=!@#$%^&* diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java index c720013125..cb038e9503 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; +import org.apache.dolphinscheduler.common.Constants; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -89,4 +90,42 @@ public class CommonUtilsTest { } Assert.assertTrue(true); } + + @Test + public void encodePassword() { + + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true"); + + Assert.assertEquals("",CommonUtils.encodePassword("")); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==",CommonUtils.encodePassword("123456")); + Assert.assertEquals("IUAjJCVeJipJVkZCV2xoVFYwQT0=",CommonUtils.encodePassword("!QAZXSW@")); + Assert.assertEquals("IUAjJCVeJipOV1JtWjJWeUtFQT0=",CommonUtils.encodePassword("5dfger(@")); + + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false"); + + Assert.assertEquals("",CommonUtils.encodePassword("")); + Assert.assertEquals("123456",CommonUtils.encodePassword("123456")); + Assert.assertEquals("!QAZXSW@",CommonUtils.encodePassword("!QAZXSW@")); + Assert.assertEquals("5dfger(@",CommonUtils.encodePassword("5dfger(@")); + + } + + @Test + public void decodePassword() { + + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true"); + + Assert.assertEquals("", CommonUtils.decodePassword("")); + Assert.assertEquals("123456", CommonUtils.decodePassword("IUAjJCVeJipNVEl6TkRVMg==")); + Assert.assertEquals("!QAZXSW@", CommonUtils.decodePassword("IUAjJCVeJipJVkZCV2xoVFYwQT0=")); + Assert.assertEquals("5dfger(@", CommonUtils.decodePassword("IUAjJCVeJipOV1JtWjJWeUtFQT0=")); + + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false"); + + Assert.assertEquals("", CommonUtils.decodePassword("")); + Assert.assertEquals("123456", CommonUtils.decodePassword("123456")); + Assert.assertEquals("!QAZXSW@", CommonUtils.decodePassword("!QAZXSW@")); + Assert.assertEquals("5dfger(@", CommonUtils.decodePassword("5dfger(@")); + } + } \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java index 96217842bf..b2a255b2e2 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; +import org.apache.dolphinscheduler.common.Constants; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -67,4 +68,17 @@ public class FileUtilsTest { Assert.assertTrue(false); } } + + @Test + public void testSetValue() { + try { + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true"); + Assert.assertTrue(PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE)); + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false"); + Assert.assertFalse(PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE)); + } catch (Exception e) { + Assert.assertTrue(false); + } + } + } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java index 90ed11aed5..aee7ac8880 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.common.utils; import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.http.impl.client.CloseableHttpClient; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -42,4 +43,11 @@ public class HttpUtilsTest { result = HttpUtils.get("https://123.333.111.33/ccc"); Assert.assertNull(result); } + + @Test + public void testGetHttpClient() { + CloseableHttpClient httpClient1 = HttpUtils.getInstance(); + CloseableHttpClient httpClient2 = HttpUtils.getInstance(); + Assert.assertEquals(httpClient1, httpClient2); + } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java index 2b9b0b54fd..eb43b40163 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.Constants; +import org.junit.Assert; import org.junit.Test; import static org.junit.Assert.assertNotNull; @@ -26,6 +27,5 @@ public class PropertyUtilsTest { @Test public void getString() { assertNotNull(PropertyUtils.getString(Constants.FS_DEFAULTFS)); - assertNotNull(PropertyUtils.getInt("spring.redis.port")); } } \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java index a8cad85b45..ccae36aae0 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java @@ -20,6 +20,7 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -182,8 +183,12 @@ public abstract class BaseDataSource { this.user = user; } + /** + * password need decode + * @return + */ public String getPassword() { - return password; + return CommonUtils.decodePassword(password); } public void setPassword(String password) { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java index 50e5e7b996..9fb756d3a7 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java @@ -82,6 +82,8 @@ public class MySQLDataSource extends BaseDataSource { @Override public String getPassword() { + // password need decode + password = super.getPassword(); if(password.contains(sensitiveParam)){ logger.warn("sensitive param : {} in password field is filtered", sensitiveParam); password = password.replace(sensitiveParam, ""); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java index 9e27d949aa..5dcdc75ab5 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java @@ -26,6 +26,8 @@ import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.dao.utils.PropertyUtils; +import org.apache.ibatis.mapping.DatabaseIdProvider; +import org.apache.ibatis.mapping.VendorDatabaseIdProvider; import org.apache.ibatis.session.SqlSession; import org.apache.ibatis.session.SqlSessionFactory; import org.apache.ibatis.type.JdbcType; @@ -39,6 +41,8 @@ import org.springframework.core.io.support.PathMatchingResourcePatternResolver; import org.springframework.core.io.support.ResourcePatternResolver; import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import java.util.Properties; + /** * data source connection factory @@ -129,6 +133,7 @@ public class SpringConnectionFactory { ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); sqlSessionFactoryBean.setMapperLocations(resolver.getResources("org/apache/dolphinscheduler/dao/mapper/*Mapper.xml")); sqlSessionFactoryBean.setTypeEnumsPackage("org.apache.dolphinscheduler.*.enums"); + sqlSessionFactoryBean.setDatabaseIdProvider(databaseIdProvider()); return sqlSessionFactoryBean.getObject(); } @@ -142,4 +147,13 @@ public class SpringConnectionFactory { return new SqlSessionTemplate(sqlSessionFactory()); } + @Bean + public DatabaseIdProvider databaseIdProvider(){ + DatabaseIdProvider databaseIdProvider = new VendorDatabaseIdProvider(); + Properties properties = new Properties(); + properties.setProperty("MySQL", "mysql"); + properties.setProperty("PostgreSQL", "pg"); + databaseIdProvider.setProperties(properties); + return databaseIdProvider; + } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowLineage.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowLineage.java new file mode 100644 index 0000000000..6c2d9c3168 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowLineage.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.dao.entity; + +import java.util.Date; + +public class WorkFlowLineage { + private int workFlowId; + private String workFlowName; + private String workFlowPublishStatus; + private Date scheduleStartTime; + private Date scheduleEndTime; + private String crontab; + private int schedulePublishStatus; + private String sourceWorkFlowId; + + public String getSourceWorkFlowId() { + return sourceWorkFlowId; + } + + public void setSourceWorkFlowId(String sourceWorkFlowId) { + this.sourceWorkFlowId = sourceWorkFlowId; + } + + public int getWorkFlowId() { + return workFlowId; + } + + public void setWorkFlowId(int workFlowId) { + this.workFlowId = workFlowId; + } + + public String getWorkFlowName() { + return workFlowName; + } + + public void setWorkFlowName(String workFlowName) { + this.workFlowName = workFlowName; + } + + public String getWorkFlowPublishStatus() { + return workFlowPublishStatus; + } + + public void setWorkFlowPublishStatus(String workFlowPublishStatus) { + this.workFlowPublishStatus = workFlowPublishStatus; + } + + public Date getScheduleStartTime() { + return scheduleStartTime; + } + + public void setScheduleStartTime(Date scheduleStartTime) { + this.scheduleStartTime = scheduleStartTime; + } + + public Date getScheduleEndTime() { + return scheduleEndTime; + } + + public void setScheduleEndTime(Date scheduleEndTime) { + this.scheduleEndTime = scheduleEndTime; + } + + public String getCrontab() { + return crontab; + } + + public void setCrontab(String crontab) { + this.crontab = crontab; + } + + public int getSchedulePublishStatus() { + return schedulePublishStatus; + } + + public void setSchedulePublishStatus(int schedulePublishStatus) { + this.schedulePublishStatus = schedulePublishStatus; + } +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowRelation.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowRelation.java new file mode 100644 index 0000000000..c03c68eb0d --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowRelation.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.dao.entity; + +public class WorkFlowRelation { + private int sourceWorkFlowId; + private int targetWorkFlowId; + + public int getSourceWorkFlowId() { + return sourceWorkFlowId; + } + + public void setSourceWorkFlowId(int sourceWorkFlowId) { + this.sourceWorkFlowId = sourceWorkFlowId; + } + + public int getTargetWorkFlowId() { + return targetWorkFlowId; + } + + public void setTargetWorkFlowId(int targetWorkFlowId) { + this.targetWorkFlowId = targetWorkFlowId; + } +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java index 5ca192811e..b7bd081cfe 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java @@ -193,4 +193,17 @@ public interface ProcessInstanceMapper extends BaseMapper { ProcessInstance queryLastManualProcess(@Param("processDefinitionId") int definitionId, @Param("startTime") Date startTime, @Param("endTime") Date endTime); + /** + * query top n process instance order by running duration + * @param size + * @param status process instance status + * @param startTime + * @param endTime + * @return ProcessInstance list + */ + List queryTopNProcessInstance(@Param("size") int size, + @Param("startTime") Date startTime, + @Param("endTime") Date endTime, + @Param("status")ExecutionStatus status); + } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapper.java new file mode 100644 index 0000000000..3abba3b7d3 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapper.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.dao.mapper; + +import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage; +import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation; +import org.apache.ibatis.annotations.Param; +import java.util.List; +import java.util.Set; + +public interface WorkFlowLineageMapper { + + public List queryByName(@Param("searchVal") String searchVal, @Param("projectId") int projectId); + + public List queryByIds(@Param("ids") Set ids, @Param("projectId") int projectId); + + public List querySourceTarget(@Param("id") int id); +} diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml index 3559ca9c85..bbc331a67d 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml @@ -37,6 +37,16 @@ order by id asc + + + select tepd.id as work_flow_id,tepd.name as work_flow_name + from t_ds_process_definition tepd + left join t_ds_schedules tes on tepd.id = tes.process_definition_id + where tepd.project_id = #{projectId} + + and tepd.name like concat('%', #{searchVal}, '%') + + + + + + + + + + + \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java index 6c44c3e329..9250e500d1 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.junit.Assert; import org.junit.Test; @@ -112,4 +114,51 @@ public class BaseDataSourceTest { } + + @Test + public void testGetPassword() { + BaseDataSource dataSource = new BaseDataSource() { + @Override + public String driverClassSelector() { + return null; + } + + @Override + public DbType dbTypeSelector() { + return null; + } + }; + + String password= ""; + dataSource.setPassword(password); + Assert.assertEquals("", dataSource.getPassword()); + password= "IUAjJCVeJipNVEl6TkRVMg=="; + dataSource.setPassword(password); + Assert.assertNotNull(dataSource.getPassword()); + Assert.assertNotNull(dataSource.getPassword()); + + dataSource.setPassword(password); + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true"); + Assert.assertEquals("123456", dataSource.getPassword()); + + dataSource.setPassword(password); + Assert.assertEquals("123456", dataSource.getPassword()); + Assert.assertEquals("123456", dataSource.getPassword()); + Assert.assertEquals("123456", dataSource.getPassword()); + + dataSource.setPassword(password); + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false"); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + + dataSource.setPassword(password); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + + + } + + + + } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSourceTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSourceTest.java index 59bf5c412e..2e9e904cd9 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSourceTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSourceTest.java @@ -16,6 +16,8 @@ */ package org.apache.dolphinscheduler.dao.datasource; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.junit.Assert; import org.junit.Test; @@ -46,6 +48,7 @@ public class MySQLDataSourceTest { Assert.assertEquals("test_pwd?", dataSource.getPassword()); } + @Test public void testFilterOther(){ MySQLDataSource dataSource = new MySQLDataSource(); @@ -61,4 +64,36 @@ public class MySQLDataSourceTest { other = dataSource.filterOther("serverTimezone=Asia/Shanghai&autoDeserialize=true&characterEncoding=utf8"); Assert.assertEquals("serverTimezone=Asia/Shanghai&characterEncoding=utf8", other); } + + @Test + public void testGetPasswordWithDecodePassword(){ + MySQLDataSource dataSource = new MySQLDataSource(); + String password= ""; + dataSource.setPassword(password); + Assert.assertEquals("", dataSource.getPassword()); + password= "IUAjJCVeJipNVEl6TkRVMg=="; + dataSource.setPassword(password); + Assert.assertNotNull(dataSource.getPassword()); + Assert.assertNotNull(dataSource.getPassword()); + + dataSource.setPassword(password); + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true"); + Assert.assertEquals("123456", dataSource.getPassword()); + + dataSource.setPassword(password); + Assert.assertEquals("123456", dataSource.getPassword()); + Assert.assertEquals("123456", dataSource.getPassword()); + Assert.assertEquals("123456", dataSource.getPassword()); + + dataSource.setPassword(password); + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false"); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + + dataSource.setPassword(password); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword()); + + } + } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java index c16ece7e85..0c1dc20ac9 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java @@ -63,7 +63,6 @@ public class AccessTokenMapperTest { Integer userId = 1; AccessToken accessToken = createAccessToken(userId); - assertNotNull(accessToken.getId()); assertThat(accessToken.getId(), greaterThan(0)); } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java index 4e298e3aa3..064305d33b 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java @@ -54,7 +54,6 @@ public class AlertMapperTest { @Test public void testInsert(){ Alert expectedAlert = createAlert(); - assertNotNull(expectedAlert.getId()); assertThat(expectedAlert.getId(), greaterThan(0)); } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java index 297ea66c94..a64ab711ad 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java @@ -62,7 +62,6 @@ public class CommandMapperTest { @Test public void testInsert(){ Command command = createCommand(); - assertNotNull(command.getId()); assertThat(command.getId(),greaterThan(0)); } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java index 92df6cc45c..04d726e357 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java @@ -71,7 +71,6 @@ public class DataSourceMapperTest { @Test public void testInsert(){ DataSource dataSource = createDataSource(); - assertNotNull(dataSource.getId()); assertThat(dataSource.getId(), greaterThan(0)); } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java index 3da6e69cce..c5bd626f9f 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java @@ -52,6 +52,25 @@ public class ProcessInstanceMapperTest { ProjectMapper projectMapper; + /** + * insert process instance with specified start time and end time,set state to SUCCESS + * + * @param startTime + * @param endTime + * @return + */ + private ProcessInstance insertOne(Date startTime, Date endTime) { + ProcessInstance processInstance = new ProcessInstance(); + Date start = startTime; + Date end = endTime; + processInstance.setStartTime(start); + processInstance.setEndTime(end); + processInstance.setState(ExecutionStatus.SUCCESS); + + processInstanceMapper.insert(processInstance); + return processInstance; + } + /** * insert * @return ProcessInstance @@ -113,7 +132,7 @@ public class ProcessInstanceMapperTest { processInstanceMapper.updateById(processInstance); ProcessInstance processInstance1 = processInstanceMapper.queryDetailById(processInstance.getId()); - Assert.assertNotEquals(processInstance1, 50); + Assert.assertNotNull(processInstance1); processInstanceMapper.deleteById(processInstance.getId()); } @@ -124,11 +143,11 @@ public class ProcessInstanceMapperTest { public void testQueryByHostAndStates() { ProcessInstance processInstance = insertOne(); processInstance.setHost("192.168.2.155"); - processInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + processInstance.setState(ExecutionStatus.RUNNING_EXECUTION); processInstanceMapper.updateById(processInstance); int[] stateArray = new int[]{ - ExecutionStatus.RUNNING_EXEUTION.ordinal(), + ExecutionStatus.RUNNING_EXECUTION.ordinal(), ExecutionStatus.SUCCESS.ordinal()}; List processInstances = processInstanceMapper.queryByHostAndStatus(null, stateArray); @@ -145,7 +164,7 @@ public class ProcessInstanceMapperTest { int[] stateArray = new int[]{ - ExecutionStatus.RUNNING_EXEUTION.ordinal(), + ExecutionStatus.RUNNING_EXECUTION.ordinal(), ExecutionStatus.SUCCESS.ordinal()}; ProcessDefinition processDefinition = new ProcessDefinition(); @@ -155,7 +174,7 @@ public class ProcessInstanceMapperTest { ProcessInstance processInstance = insertOne(); processInstance.setProcessDefinitionId(processDefinition.getId()); - processInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + processInstance.setState(ExecutionStatus.RUNNING_EXECUTION); processInstance.setIsSubProcess(Flag.NO); processInstance.setStartTime(new Date()); @@ -188,12 +207,12 @@ public class ProcessInstanceMapperTest { public void testSetFailoverByHostAndStateArray() { int[] stateArray = new int[]{ - ExecutionStatus.RUNNING_EXEUTION.ordinal(), + ExecutionStatus.RUNNING_EXECUTION.ordinal(), ExecutionStatus.SUCCESS.ordinal()}; ProcessInstance processInstance = insertOne(); - processInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + processInstance.setState(ExecutionStatus.RUNNING_EXECUTION); processInstance.setHost("192.168.2.220"); processInstanceMapper.updateById(processInstance); String host = processInstance.getHost(); @@ -214,9 +233,9 @@ public class ProcessInstanceMapperTest { ProcessInstance processInstance = insertOne(); - processInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + processInstance.setState(ExecutionStatus.RUNNING_EXECUTION); processInstanceMapper.updateById(processInstance); - processInstanceMapper.updateProcessInstanceByState(ExecutionStatus.RUNNING_EXEUTION, ExecutionStatus.SUCCESS); + processInstanceMapper.updateProcessInstanceByState(ExecutionStatus.RUNNING_EXECUTION, ExecutionStatus.SUCCESS); ProcessInstance processInstance1 = processInstanceMapper.selectById(processInstance.getId()); @@ -294,11 +313,11 @@ public class ProcessInstanceMapperTest { @Test public void testQueryLastRunningProcess() { ProcessInstance processInstance = insertOne(); - processInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + processInstance.setState(ExecutionStatus.RUNNING_EXECUTION); processInstanceMapper.updateById(processInstance); int[] stateArray = new int[]{ - ExecutionStatus.RUNNING_EXEUTION.ordinal(), + ExecutionStatus.RUNNING_EXECUTION.ordinal(), ExecutionStatus.SUBMITTED_SUCCESS.ordinal()}; ProcessInstance processInstance1 = processInstanceMapper.queryLastRunningProcess(processInstance.getProcessDefinitionId(), null, null , stateArray); @@ -329,4 +348,50 @@ public class ProcessInstanceMapperTest { processInstanceMapper.deleteById(processInstance.getId()); } + + + /** + * test whether it is in descending order by running duration + * + * @param processInstances + * @return + */ + private boolean isSortedByDuration(List processInstances) { + for (int i = 1; i < processInstances.size(); i++) { + long d1 = processInstances.get(i).getEndTime().getTime() - processInstances.get(i).getStartTime().getTime(); + long d2 = processInstances.get(i - 1).getEndTime().getTime() - processInstances.get(i - 1).getStartTime().getTime(); + if (d1 > d2) { + return false; + } + } + return true; + } + + /** + * test query top n process instance order by running duration + */ + @Test + public void testQueryTopNProcessInstance() { + Date startTime1 = new Date(2019, 7, 9, 10, 9, 9); + Date endTime1 = new Date(2019, 7, 9, 10, 9, 14); + Date startTime2 = new Date(2020, 7, 9, 10, 9, 9); + Date endTime2 = new Date(2020, 7, 9, 10, 9, 30); + Date startTime3 = new Date(2020, 6, 9, 10, 9, 9); + Date endTime3 = new Date(2020, 7, 9, 10, 9, 30); + ProcessInstance processInstance1 = insertOne(startTime1, endTime1); + ProcessInstance processInstance2 = insertOne(startTime2, endTime2); + ProcessInstance processInstance3 = insertOne(startTime3, endTime3); + Date start = new Date(2020, 1, 1, 1, 1, 1); + Date end = new Date(2021, 1, 1, 1, 1, 1); + List processInstances = processInstanceMapper.queryTopNProcessInstance(2, start, end,ExecutionStatus.SUCCESS); + Assert.assertEquals(2, processInstances.size()); + Assert.assertTrue(isSortedByDuration(processInstances)); + for (ProcessInstance processInstance : processInstances) { + Assert.assertTrue(processInstance.getState().typeIsSuccess()); + } + processInstanceMapper.deleteById(processInstance1.getId()); + processInstanceMapper.deleteById(processInstance2.getId()); + processInstanceMapper.deleteById(processInstance3.getId()); + + } } \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java index b224067a29..017527137b 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java @@ -64,7 +64,7 @@ public class TaskInstanceMapperTest { TaskInstance taskInstance = new TaskInstance(); taskInstance.setFlag(Flag.YES); taskInstance.setName("ut task"); - taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION); taskInstance.setStartTime(new Date()); taskInstance.setEndTime(new Date()); taskInstance.setTaskJson("{}"); @@ -118,7 +118,7 @@ public class TaskInstanceMapperTest { taskInstanceMapper.updateById(task); List taskInstances = taskInstanceMapper.queryTaskByProcessIdAndState( task.getProcessInstanceId(), - ExecutionStatus.RUNNING_EXEUTION.ordinal() + ExecutionStatus.RUNNING_EXECUTION.ordinal() ); taskInstanceMapper.deleteById(task.getId()); Assert.assertNotEquals(taskInstances.size(), 0); @@ -162,7 +162,7 @@ public class TaskInstanceMapperTest { taskInstanceMapper.updateById(task); List taskInstances = taskInstanceMapper.queryByHostAndStatus( - task.getHost(), new int[]{ExecutionStatus.RUNNING_EXEUTION.ordinal()} + task.getHost(), new int[]{ExecutionStatus.RUNNING_EXECUTION.ordinal()} ); taskInstanceMapper.deleteById(task.getId()); Assert.assertNotEquals(taskInstances.size(), 0); @@ -179,7 +179,7 @@ public class TaskInstanceMapperTest { int setResult = taskInstanceMapper.setFailoverByHostAndStateArray( task.getHost(), - new int[]{ExecutionStatus.RUNNING_EXEUTION.ordinal()}, + new int[]{ExecutionStatus.RUNNING_EXECUTION.ordinal()}, ExecutionStatus.NEED_FAULT_TOLERANCE ); taskInstanceMapper.deleteById(task.getId()); @@ -268,7 +268,7 @@ public class TaskInstanceMapperTest { ProcessInstance processInstance = new ProcessInstance(); processInstance.setProcessDefinitionId(definition.getId()); - processInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + processInstance.setState(ExecutionStatus.RUNNING_EXECUTION); processInstance.setName("ut process"); processInstance.setStartTime(new Date()); processInstance.setEndTime(new Date()); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapperTest.java new file mode 100644 index 0000000000..5645e1cf75 --- /dev/null +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapperTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.dao.mapper; + +import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage; +import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.Rollback; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.transaction.annotation.Transactional; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +@RunWith(SpringRunner.class) +@SpringBootTest +@Transactional +@Rollback(true) +public class WorkFlowLineageMapperTest { + @Autowired + private WorkFlowLineageMapper workFlowLineageMapper; + + @Test + public void testQueryByName() { + List workFlowLineages = workFlowLineageMapper.queryByName("test",1); + Assert.assertNotEquals(workFlowLineages.size(), 0); + } + + + @Test + public void testQueryByIds() { + Set ids = new HashSet<>(); + ids.add(1); + List workFlowLineages = workFlowLineageMapper.queryByIds(ids,1); + Assert.assertNotEquals(workFlowLineages.size(), 0); + } + + @Test + public void testQuerySourceTarget() { + List workFlowRelations = workFlowLineageMapper.querySourceTarget(1); + Assert.assertNotEquals(workFlowRelations.size(), 0); + } +} diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml index ca3834e90b..b193dd85c8 100644 --- a/dolphinscheduler-dist/pom.xml +++ b/dolphinscheduler-dist/pom.xml @@ -318,6 +318,7 @@ start-all.sh stop-all.sh dolphinscheduler-daemon.sh + status-all.sh diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml index 28bbb361cd..9b61c5e094 100644 --- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml +++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml @@ -189,6 +189,7 @@ start-all.sh stop-all.sh dolphinscheduler-daemon.sh + status-all.sh ./bin diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml index f41151cd09..b47a0ae393 100644 --- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml +++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml @@ -197,6 +197,7 @@ start-all.sh stop-all.sh dolphinscheduler-daemon.sh + status-all.sh ./bin diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index d86374244f..e6c7792388 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -123,8 +123,8 @@ public class MasterServer { // self tolerant this.zkMasterClient.start(); - // - masterSchedulerService.start(); + // scheduler start + this.masterSchedulerService.start(); // start QuartzExecutors // what system should do if exception diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java index b9772ca523..7abb31b31c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java @@ -89,6 +89,7 @@ public class TaskResponseService { eventQueue.put(taskResponseEvent); } catch (InterruptedException e) { logger.error("put task : {} error :{}", taskResponseEvent,e); + Thread.currentThread().interrupt(); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java index 826a7c8fb9..a410f99fc1 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java @@ -124,7 +124,7 @@ public class ConditionsTaskExecThread extends MasterBaseTaskExecThread { private void initTaskParameters() { this.taskInstance.setLogPath(getTaskLogPath(taskInstance)); this.taskInstance.setHost(NetUtils.getHost() + Constants.COLON + masterConfig.getListenPort()); - taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION); taskInstance.setStartTime(new Date()); this.processService.saveTaskInstance(taskInstance); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java index a67e24bcf2..183f1aac42 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java @@ -173,7 +173,7 @@ public class DependentTaskExecThread extends MasterBaseTaskExecThread { private void initTaskParameters() { taskInstance.setLogPath(getTaskLogPath(taskInstance)); taskInstance.setHost(NetUtils.getHost() + Constants.COLON + masterConfig.getListenPort()); - taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION); taskInstance.setStartTime(new Date()); processService.updateTaskInstance(taskInstance); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java index c903dbc6b0..aac201c403 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java @@ -172,7 +172,7 @@ public class MasterBaseTaskExecThread implements Callable { return true; } // task cannot submit when running - if(taskInstance.getState() == ExecutionStatus.RUNNING_EXEUTION){ + if(taskInstance.getState() == ExecutionStatus.RUNNING_EXECUTION){ logger.info(String.format("submit to task, but task [%s] state already be running. ", taskInstance.getName())); return true; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index df1e139760..541c677b21 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -35,7 +35,6 @@ import org.apache.dolphinscheduler.dao.utils.DagHelper; import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.utils.AlertManager; -import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; import org.slf4j.Logger; @@ -123,12 +122,12 @@ public class MasterExecThread implements Runnable { /** * alert manager */ - private AlertManager alertManager = new AlertManager(); + private AlertManager alertManager; /** * the object of DAG */ - private DAG dag; + private DAG dag; /** * process service @@ -151,15 +150,20 @@ public class MasterExecThread implements Runnable { * @param processService processService * @param nettyRemotingClient nettyRemotingClient */ - public MasterExecThread(ProcessInstance processInstance, ProcessService processService, NettyRemotingClient nettyRemotingClient){ + public MasterExecThread(ProcessInstance processInstance + , ProcessService processService + , NettyRemotingClient nettyRemotingClient + , AlertManager alertManager + , MasterConfig masterConfig) { this.processService = processService; this.processInstance = processInstance; - this.masterConfig = SpringApplicationContext.getBean(MasterConfig.class); + this.masterConfig = masterConfig; int masterTaskExecNum = masterConfig.getMasterExecTaskNum(); this.taskExecService = ThreadUtils.newDaemonFixedThreadExecutor("Master-Task-Exec-Thread", masterTaskExecNum); this.nettyRemotingClient = nettyRemotingClient; + this.alertManager = alertManager; } @@ -248,6 +252,9 @@ public class MasterExecThread implements Runnable { } while(Stopper.isRunning()){ + + logger.info("process {} start to complement {} data", + processInstance.getId(), DateUtils.dateToString(scheduleDate)); // prepare dag and other info prepareProcess(); @@ -262,13 +269,13 @@ public class MasterExecThread implements Runnable { // execute process ,waiting for end runProcess(); + endProcess(); // process instance failure ，no more complements if(!processInstance.getState().typeIsSuccess()){ logger.info("process {} state {}, complement not completely!", processInstance.getId(), processInstance.getState()); break; } - // current process instance success ,next execute if(null == iterator){ // loop by day @@ -287,9 +294,7 @@ public class MasterExecThread implements Runnable { } scheduleDate = iterator.next(); } - - logger.info("process {} start to complement {} data", - processInstance.getId(), DateUtils.dateToString(scheduleDate)); + // flow end // execute next process instance complement data processInstance.setScheduleTime(scheduleDate); if(cmdParam.containsKey(Constants.CMDPARAM_RECOVERY_START_NODE_STRING)){ @@ -297,23 +302,16 @@ public class MasterExecThread implements Runnable { processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam)); } - List taskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId()); - for(TaskInstance taskInstance : taskInstanceList){ - taskInstance.setFlag(Flag.NO); - processService.updateTaskInstance(taskInstance); - } - processInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + processInstance.setState(ExecutionStatus.RUNNING_EXECUTION); processInstance.setGlobalParams(ParameterUtils.curingGlobalParams( processInstance.getProcessDefinition().getGlobalParamMap(), processInstance.getProcessDefinition().getGlobalParamList(), CommandType.COMPLEMENT_DATA, processInstance.getScheduleTime())); - + processInstance.setId(0); + processInstance.setStartTime(new Date()); + processInstance.setEndTime(null); processService.saveProcessInstance(processInstance); } - - // flow end - endProcess(); - } @@ -738,7 +736,7 @@ public class MasterExecThread implements Runnable { // if the running task is not completed, the state remains unchanged return state; }else{ - return ExecutionStatus.RUNNING_EXEUTION; + return ExecutionStatus.RUNNING_EXECUTION; } } @@ -815,7 +813,7 @@ public class MasterExecThread implements Runnable { ProcessInstance instance = processService.findProcessInstanceById(processInstance.getId()); ExecutionStatus state = instance.getState(); - if(activeTaskNode.size() > 0 || retryTaskExists()){ + if(activeTaskNode.size() > 0 || hasRetryTaskInStandBy()){ // active task and retry task exists return runningState(state); } @@ -848,11 +846,11 @@ public class MasterExecThread implements Runnable { } // success - if(state == ExecutionStatus.RUNNING_EXEUTION){ + if(state == ExecutionStatus.RUNNING_EXECUTION){ List killTasks = getCompleteTaskByState(ExecutionStatus.KILL); if(readyToSubmitTaskList.size() > 0){ //tasks currently pending submission, no retries, indicating that depend is waiting to complete - return ExecutionStatus.RUNNING_EXEUTION; + return ExecutionStatus.RUNNING_EXECUTION; }else if(CollectionUtils.isNotEmpty(killTasks)){ // tasks maybe killed manually return ExecutionStatus.FAILURE; @@ -865,24 +863,6 @@ public class MasterExecThread implements Runnable { return state; } - /** - * whether standby task list have retry tasks - * @return - */ - private boolean retryTaskExists() { - - boolean result = false; - - for(String taskName : readyToSubmitTaskList.keySet()){ - TaskInstance task = readyToSubmitTaskList.get(taskName); - if(task.getState().typeIsFailure()){ - result = true; - break; - } - } - return result; - } - /** * whether complement end * @return Boolean whether is complement end @@ -1057,6 +1037,7 @@ public class MasterExecThread implements Runnable { Thread.sleep(Constants.SLEEP_TIME_MILLIS); } catch (InterruptedException e) { logger.error(e.getMessage(),e); + Thread.currentThread().interrupt(); } updateProcessInstanceState(); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java index b0c0ccb2df..30dd0f9f12 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java @@ -16,6 +16,11 @@ */ package org.apache.dolphinscheduler.server.master.runner; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import javax.annotation.PostConstruct; + import org.apache.curator.framework.imps.CuratorFrameworkState; import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.apache.dolphinscheduler.common.Constants; @@ -28,6 +33,7 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; import org.apache.dolphinscheduler.server.master.config.MasterConfig; +import org.apache.dolphinscheduler.server.utils.AlertManager; import org.apache.dolphinscheduler.server.zk.ZKMasterClient; import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; @@ -35,10 +41,6 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import javax.annotation.PostConstruct; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - /** * master scheduler thread */ @@ -68,6 +70,11 @@ public class MasterSchedulerService extends Thread { @Autowired private MasterConfig masterConfig; + /** + * alert manager + */ + private AlertManager alertManager = new AlertManager(); + /** * netty remoting client */ @@ -90,7 +97,7 @@ public class MasterSchedulerService extends Thread { } @Override - public void start(){ + public synchronized void start(){ super.setName("MasterSchedulerService"); super.start(); } @@ -100,7 +107,9 @@ public class MasterSchedulerService extends Thread { boolean terminated = false; try { terminated = masterExecService.awaitTermination(5, TimeUnit.SECONDS); - } catch (InterruptedException ignore) {} + } catch (InterruptedException ignore) { + Thread.currentThread().interrupt(); + } if(!terminated){ logger.warn("masterExecService shutdown without terminated, increase await time"); } @@ -139,7 +148,13 @@ public class MasterSchedulerService extends Thread { this.masterConfig.getMasterExecThreads() - activeCount, command); if (processInstance != null) { logger.info("start master exec thread , split DAG ..."); - masterExecService.execute(new MasterExecThread(processInstance, processService, nettyRemotingClient)); + masterExecService.execute( + new MasterExecThread( + processInstance + , processService + , nettyRemotingClient + , alertManager + , masterConfig)); } }catch (Exception e){ logger.error("scan command error ", e); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java index ee290487b7..2b1508dc44 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java @@ -93,7 +93,7 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread { return false; } - taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION); taskInstance.setStartTime(new Date()); processService.updateTaskInstance(taskInstance); return true; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java index 788a1d7119..2ce4515279 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java @@ -167,7 +167,7 @@ public class TaskExecuteProcessor implements NettyRequestProcessor { private TaskExecuteAckCommand buildAckCommand(TaskExecutionContext taskExecutionContext) { TaskExecuteAckCommand ackCommand = new TaskExecuteAckCommand(); ackCommand.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); - ackCommand.setStatus(ExecutionStatus.RUNNING_EXEUTION.getCode()); + ackCommand.setStatus(ExecutionStatus.RUNNING_EXECUTION.getCode()); ackCommand.setLogPath(getTaskLogPath(taskExecutionContext)); ackCommand.setHost(taskExecutionContext.getHost()); ackCommand.setStartTime(new Date()); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java index 19ba9c9a21..f98d451350 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java @@ -50,6 +50,8 @@ public class TaskManager { switch (EnumUtils.getEnum(TaskType.class,taskExecutionContext.getTaskType())) { case SHELL: return new ShellTask(taskExecutionContext, logger); + case WATERDROP: + return new ShellTask(taskExecutionContext, logger); case PROCEDURE: return new ProcedureTask(taskExecutionContext, logger); case SQL: diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf old mode 100644 new mode 100755 index 6da5fbed55..a20fa66087 --- a/dolphinscheduler-server/src/main/resources/config/install_config.conf +++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf @@ -38,6 +38,9 @@ dbname="dolphinscheduler" # zk cluster zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181" +# zk root directory +zkRoot="/dolphinscheduler" + # Note: the target installation path for dolphinscheduler, please not config as the same as the current path (pwd) installPath="/data1_1T/dolphinscheduler" @@ -138,4 +141,4 @@ alertServer="ds3" # run api machine # note: list of machine hostnames for deploying api server -apiServers="ds1" +apiServers="ds1" \ No newline at end of file diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java index 299d4ba800..f1ee8ccf11 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java @@ -124,7 +124,7 @@ public class ConditionsTaskTest { ProcessInstance processInstance = new ProcessInstance(); processInstance.setId(10112); processInstance.setProcessDefinitionId(100001); - processInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + processInstance.setState(ExecutionStatus.RUNNING_EXECUTION); return processInstance; } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java index 66bc3afe84..351be8aa65 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java @@ -176,7 +176,7 @@ public class DependentTaskTest { ProcessInstance processInstance = new ProcessInstance(); processInstance.setId(10111); processInstance.setProcessDefinitionId(0); - processInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + processInstance.setState(ExecutionStatus.RUNNING_EXECUTION); return processInstance; } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java index 5875d21758..bf1e7e2d7a 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java @@ -72,8 +72,6 @@ public class MasterExecThreadTest { applicationContext = mock(ApplicationContext.class); config = new MasterConfig(); config.setMasterExecTaskNum(1); - SpringApplicationContext springApplicationContext = new SpringApplicationContext(); - springApplicationContext.setApplicationContext(applicationContext); Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config); processInstance = mock(ProcessInstance.class); @@ -84,14 +82,17 @@ public class MasterExecThreadTest { Mockito.when(processInstance.getScheduleTime()).thenReturn(DateUtils.stringToDate("2020-01-01 00:00:00")); Map cmdParam = new HashMap<>(); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, "2020-01-01 00:00:00"); - cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, "2020-01-31 23:00:00"); + cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, "2020-01-20 23:00:00"); Mockito.when(processInstance.getCommandParam()).thenReturn(JSONUtils.toJsonString(cmdParam)); ProcessDefinition processDefinition = new ProcessDefinition(); processDefinition.setGlobalParamMap(Collections.EMPTY_MAP); processDefinition.setGlobalParamList(Collections.EMPTY_LIST); Mockito.when(processInstance.getProcessDefinition()).thenReturn(processDefinition); - masterExecThread = PowerMockito.spy(new MasterExecThread(processInstance, processService,null)); + masterExecThread = PowerMockito.spy(new MasterExecThread( + processInstance + , processService + ,null, null, config)); // prepareProcess init dag Field dag = MasterExecThread.class.getDeclaredField("dag"); dag.setAccessible(true); @@ -114,11 +115,11 @@ public class MasterExecThreadTest { Method method = MasterExecThread.class.getDeclaredMethod("executeComplementProcess"); method.setAccessible(true); method.invoke(masterExecThread); - // one create save, and 1-30 for next save, and last day 31 no save - verify(processService, times(31)).saveProcessInstance(processInstance); + // one create save, and 1-30 for next save, and last day 20 no save + verify(processService, times(20)).saveProcessInstance(processInstance); }catch (Exception e){ e.printStackTrace(); - Assert.assertTrue(false); + Assert.fail(); } } @@ -133,10 +134,10 @@ public class MasterExecThreadTest { Method method = MasterExecThread.class.getDeclaredMethod("executeComplementProcess"); method.setAccessible(true); method.invoke(masterExecThread); - // one create save, and 15(1 to 31 step 2) for next save, and last day 31 no save - verify(processService, times(15)).saveProcessInstance(processInstance); + // one create save, and 9(1 to 20 step 2) for next save, and last day 31 no save + verify(processService, times(9)).saveProcessInstance(processInstance); }catch (Exception e){ - Assert.assertTrue(false); + Assert.fail(); } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java index beefedb36c..997c129573 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java @@ -17,11 +17,18 @@ package org.apache.dolphinscheduler.server.master.consumer; +import java.util.Date; + import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.Priority; -import org.apache.dolphinscheduler.dao.entity.*; +import org.apache.dolphinscheduler.common.thread.Stopper; +import org.apache.dolphinscheduler.dao.entity.DataSource; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.dispatch.ExecutorDispatcher; import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager; @@ -32,8 +39,10 @@ import org.apache.dolphinscheduler.server.zk.SpringZKServer; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue; +import org.apache.dolphinscheduler.service.zk.CuratorZookeeperClient; import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -42,11 +51,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import java.util.Date; - @RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes={DependencyConfig.class, SpringApplicationContext.class, SpringZKServer.class, +@ContextConfiguration(classes={DependencyConfig.class, SpringApplicationContext.class, SpringZKServer.class, CuratorZookeeperClient.class, NettyExecutorManager.class, ExecutorDispatcher.class, ZookeeperRegistryCenter.class, TaskPriorityQueueConsumer.class, ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, MasterConfig.class}) public class TaskPriorityQueueConsumerTest { @@ -250,5 +257,11 @@ public class TaskPriorityQueueConsumerTest { taskPriorityQueueConsumer.taskInstanceIsFinalState(1); } + @After + public void close() { + Stopper.stop(); + } + + } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java index 958df01cf8..98231bee06 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java @@ -78,5 +78,7 @@ public class ExecutorDispatcherTest { ExecutionContext executionContext = ExecutionContextTestUtils.getExecutionContext(port); executorDispatcher.dispatch(executionContext); + + workerRegistry.unRegistry(); } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java index b70cf6d3e8..c38c9d4ae8 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java @@ -74,5 +74,6 @@ public class RoundRobinHostManagerTest { Host host = roundRobinHostManager.select(context); Assert.assertTrue(StringUtils.isNotEmpty(host.getAddress())); Assert.assertTrue(host.getAddress().equalsIgnoreCase(NetUtils.getHost() + ":" + workerConfig.getListenPort())); + workerRegistry.unRegistry(); } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java index dcba83271c..f19e2b4b64 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java @@ -43,7 +43,7 @@ public class TaskResponseServiceTest { @Test public void testAdd(){ - TaskResponseEvent taskResponseEvent = TaskResponseEvent.newAck(ExecutionStatus.RUNNING_EXEUTION, new Date(), + TaskResponseEvent taskResponseEvent = TaskResponseEvent.newAck(ExecutionStatus.RUNNING_EXECUTION, new Date(), "", "", "", 1); taskResponseService.addResponse(taskResponseEvent); Assert.assertTrue(taskResponseService.getEventQueue().size() == 1); @@ -57,7 +57,7 @@ public class TaskResponseServiceTest { @Test public void testStop(){ - TaskResponseEvent taskResponseEvent = TaskResponseEvent.newAck(ExecutionStatus.RUNNING_EXEUTION, new Date(), + TaskResponseEvent taskResponseEvent = TaskResponseEvent.newAck(ExecutionStatus.RUNNING_EXECUTION, new Date(), "", "", "", 1); taskResponseService.addResponse(taskResponseEvent); taskResponseService.stop(); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryTest.java index 9d90f20706..18c8b496d7 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryTest.java @@ -58,6 +58,7 @@ public class MasterRegistryTest { String masterNodePath = masterPath + "/" + (Constants.LOCAL_ADDRESS + ":" + masterConfig.getListenPort()); String heartbeat = zookeeperRegistryCenter.getZookeeperCachedOperator().get(masterNodePath); Assert.assertEquals(HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH, heartbeat.split(",").length); + masterRegistry.unRegistry(); } @Test diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java index 83095a371a..f29691e9bb 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java @@ -17,65 +17,75 @@ package org.apache.dolphinscheduler.server.master.runner; +import java.util.HashSet; +import java.util.Set; + import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.server.master.consumer.TaskPriorityQueueConsumer; -import org.apache.dolphinscheduler.server.master.dispatch.ExecutorDispatcher; -import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager; -import org.apache.dolphinscheduler.server.registry.DependencyConfig; -import org.apache.dolphinscheduler.server.registry.ZookeeperNodeManager; import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; -import org.apache.dolphinscheduler.server.zk.SpringZKServer; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; -import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; -import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; +import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; import org.springframework.context.ApplicationContext; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import java.util.HashSet; -import java.util.Set; +import com.google.common.collect.Sets; -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes={DependencyConfig.class, SpringApplicationContext.class, SpringZKServer.class, - NettyExecutorManager.class, ExecutorDispatcher.class, ZookeeperRegistryCenter.class, TaskPriorityQueueConsumer.class, - ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class}) +@RunWith(MockitoJUnitRunner.Silent.class) +@PrepareForTest(MasterTaskExecThread.class) public class MasterTaskExecThreadTest { + private MasterTaskExecThread masterTaskExecThread; + + private SpringApplicationContext springApplicationContext; + + private ZookeeperRegistryCenter zookeeperRegistryCenter; + + @Before + public void setUp() { + + ApplicationContext applicationContext = PowerMockito.mock(ApplicationContext.class); + this.springApplicationContext = new SpringApplicationContext(); + springApplicationContext.setApplicationContext(applicationContext); + this.zookeeperRegistryCenter = PowerMockito.mock(ZookeeperRegistryCenter.class); + PowerMockito.when(SpringApplicationContext.getBean(ZookeeperRegistryCenter.class)) + .thenReturn(this.zookeeperRegistryCenter); + this.masterTaskExecThread = new MasterTaskExecThread(null); + } @Test public void testExistsValidWorkerGroup1(){ - ZookeeperRegistryCenter zookeeperRegistryCenter = Mockito.mock(ZookeeperRegistryCenter.class); - Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(null); - MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(null); - masterTaskExecThread.existsValidWorkerGroup("default"); + + Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(Sets.newHashSet()); + boolean b = masterTaskExecThread.existsValidWorkerGroup("default"); + Assert.assertFalse(b); } @Test public void testExistsValidWorkerGroup2(){ - ZookeeperRegistryCenter zookeeperRegistryCenter = Mockito.mock(ZookeeperRegistryCenter.class); Set workerGorups = new HashSet<>(); workerGorups.add("test1"); workerGorups.add("test2"); Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(workerGorups); - MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(null); - masterTaskExecThread.existsValidWorkerGroup("default"); + boolean b = masterTaskExecThread.existsValidWorkerGroup("default"); + Assert.assertFalse(b); } @Test public void testExistsValidWorkerGroup3(){ - ZookeeperRegistryCenter zookeeperRegistryCenter = Mockito.mock(ZookeeperRegistryCenter.class); Set workerGorups = new HashSet<>(); workerGorups.add("test1"); Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(workerGorups); Mockito.when(zookeeperRegistryCenter.getWorkerGroupNodesDirectly("test1")).thenReturn(workerGorups); - MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(null); - masterTaskExecThread.existsValidWorkerGroup("test1"); + boolean b = masterTaskExecThread.existsValidWorkerGroup("test1"); + Assert.assertTrue(b); } @Test @@ -83,17 +93,15 @@ public class MasterTaskExecThreadTest { ProcessService processService = Mockito.mock(ProcessService.class); - ApplicationContext applicationContext = Mockito.mock(ApplicationContext.class); - SpringApplicationContext springApplicationContext = new SpringApplicationContext(); - springApplicationContext.setApplicationContext(applicationContext); - Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService); + Mockito.when(this.springApplicationContext.getBean(ProcessService.class)) + .thenReturn(processService); TaskInstance taskInstance = getTaskInstance(); Mockito.when(processService.findTaskInstanceById(252612)) .thenReturn(taskInstance); Mockito.when(processService.updateTaskInstance(taskInstance)) - .thenReturn(true); + .thenReturn(true); MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(taskInstance); masterTaskExecThread.pauseTask(); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManagerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManagerTest.java index 0e780a59ce..f6eb861069 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManagerTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManagerTest.java @@ -17,6 +17,9 @@ package org.apache.dolphinscheduler.server.registry; +import java.util.Map; +import java.util.Set; + import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.server.master.config.MasterConfig; @@ -24,6 +27,7 @@ import org.apache.dolphinscheduler.server.master.registry.MasterRegistry; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistry; import org.apache.dolphinscheduler.server.zk.SpringZKServer; +import org.apache.dolphinscheduler.service.zk.CuratorZookeeperClient; import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; import org.junit.Assert; @@ -33,16 +37,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import java.util.Map; -import java.util.Set; - /** * zookeeper node manager test */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes={DependencyConfig.class, SpringZKServer.class, MasterRegistry.class,WorkerRegistry.class, ZookeeperRegistryCenter.class, MasterConfig.class, WorkerConfig.class, - ZookeeperCachedOperator.class, ZookeeperConfig.class, ZookeeperNodeManager.class}) + ZookeeperCachedOperator.class, ZookeeperConfig.class, ZookeeperNodeManager.class, CuratorZookeeperClient.class}) public class ZookeeperNodeManagerTest { @Autowired @@ -75,6 +76,7 @@ public class ZookeeperNodeManagerTest { Assert.assertTrue(CollectionUtils.isNotEmpty(masterNodes)); Assert.assertEquals(1, masterNodes.size()); Assert.assertEquals(NetUtils.getHost() + ":" + masterConfig.getListenPort(), masterNodes.iterator().next()); + workerRegistry.unRegistry(); } @Test @@ -88,6 +90,7 @@ public class ZookeeperNodeManagerTest { Map> workerGroupNodes = zookeeperNodeManager.getWorkerGroupNodes(); Assert.assertEquals(1, workerGroupNodes.size()); Assert.assertEquals("default".trim(), workerGroupNodes.keySet().iterator().next()); + workerRegistry.unRegistry(); } @Test @@ -103,5 +106,6 @@ public class ZookeeperNodeManagerTest { Assert.assertTrue(CollectionUtils.isNotEmpty(workerNodes)); Assert.assertEquals(1, workerNodes.size()); Assert.assertEquals(NetUtils.getHost() + ":" + workerConfig.getListenPort(), workerNodes.iterator().next()); + workerRegistry.unRegistry(); } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTest.java index d09d399bef..8d58459e85 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTest.java @@ -16,7 +16,8 @@ */ package org.apache.dolphinscheduler.server.worker.processor; -import io.netty.channel.Channel; +import java.util.Date; + import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.NettyRemotingClient; @@ -40,6 +41,7 @@ import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistry; import org.apache.dolphinscheduler.server.zk.SpringZKServer; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.zk.CuratorZookeeperClient; import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; import org.junit.Assert; @@ -50,8 +52,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import java.io.IOException; -import java.util.Date; +import io.netty.channel.Channel; /** * test task call back service @@ -73,7 +74,8 @@ import java.util.Date; TaskResponseService.class, TaskAckProcessor.class, TaskResponseProcessor.class, - TaskExecuteProcessor.class}) + TaskExecuteProcessor.class, + CuratorZookeeperClient.class}) public class TaskCallbackServiceTest { @Autowired @@ -189,6 +191,7 @@ public class TaskCallbackServiceTest { nettyRemotingServer.close(); nettyRemotingClient.close(); + masterRegistry.unRegistry(); } @Test diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryTest.java index 7fc9d2bf79..0490d934e6 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryTest.java @@ -143,6 +143,7 @@ public class WorkerRegistryTest { Assert.assertEquals(0, testWorkerGroupPathZkChildren.size()); Assert.assertEquals(0, defaultWorkerGroupPathZkChildren.size()); + workerRegistry.unRegistry(); } @Test diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogPromise.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogPromise.java index 98ee3fdbbf..016176764f 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogPromise.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogPromise.java @@ -100,6 +100,7 @@ public class LogPromise { try { latch.await(timeout, TimeUnit.MILLISECONDS); } catch (InterruptedException ignore) { + Thread.currentThread().interrupt(); } PROMISES.remove(opaque); return this.result; diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 769fe0928a..7df56c6cdd 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -37,11 +37,8 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; - -import java.io.File; import java.util.*; import java.util.stream.Collectors; - import static java.util.stream.Collectors.toSet; import static org.apache.dolphinscheduler.common.Constants.*; @@ -54,7 +51,7 @@ public class ProcessService { private final Logger logger = LoggerFactory.getLogger(getClass()); private final int[] stateArray = new int[]{ExecutionStatus.SUBMITTED_SUCCESS.ordinal(), - ExecutionStatus.RUNNING_EXEUTION.ordinal(), + ExecutionStatus.RUNNING_EXECUTION.ordinal(), ExecutionStatus.READY_PAUSE.ordinal(), ExecutionStatus.READY_STOP.ordinal()}; @@ -107,7 +104,7 @@ public class ProcessService { * @param command found command * @return process instance */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public ProcessInstance handleCommand(Logger logger, String host, int validThreadNum, Command command) { ProcessInstance processInstance = constructProcessInstance(command, host); //cannot construct process instance, return null; @@ -133,7 +130,7 @@ public class ProcessService { * @param command command * @param message message */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public void moveToErrorCommand(Command command, String message) { ErrorCommand errorCommand = new ErrorCommand(command, message); this.errorCommandMapper.insert(errorCommand); @@ -454,7 +451,7 @@ public class ProcessService { Command command, Map cmdParam){ ProcessInstance processInstance = new ProcessInstance(processDefinition); - processInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + processInstance.setState(ExecutionStatus.RUNNING_EXECUTION); processInstance.setRecovery(Flag.NO); processInstance.setStartTime(new Date()); processInstance.setRunTimes(1); @@ -616,7 +613,7 @@ public class ProcessService { } processInstance.setHost(host); - ExecutionStatus runStatus = ExecutionStatus.RUNNING_EXEUTION; + ExecutionStatus runStatus = ExecutionStatus.RUNNING_EXECUTION; int runTime = processInstance.getRunTimes(); switch (commandType){ case START_PROCESS: @@ -827,7 +824,7 @@ public class ProcessService { * @param taskInstance taskInstance * @return task instance */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public TaskInstance submitTask(TaskInstance taskInstance){ ProcessInstance processInstance = this.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); logger.info("start submit task : {}, instance id:{}, state: {}", @@ -1068,7 +1065,7 @@ public class ProcessService { // running or killed // the task already exists in task queue // return state - state == ExecutionStatus.RUNNING_EXEUTION + state == ExecutionStatus.RUNNING_EXECUTION || state == ExecutionStatus.KILL || checkTaskExistsInTaskQueue(taskInstance) ){ @@ -1477,7 +1474,7 @@ public class ProcessService { * process need failover process instance * @param processInstance processInstance */ - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = RuntimeException.class) public void processNeedFailoverProcessInstances(ProcessInstance processInstance){ //1 update processInstance host is null processInstance.setHost(Constants.NULL); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java index 8b6392675a..8a7d891c2e 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java @@ -326,7 +326,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator { @Override public String toString() { return "AbstractZKClient{" + - "zkClient=" + zkClient + + "zkClient=" + getZkClient() + ", deadServerZNodeParentPath='" + getZNodeParentPath(ZKNodeType.DEAD_SERVER) + '\'' + ", masterZNodeParentPath='" + getZNodeParentPath(ZKNodeType.MASTER) + '\'' + ", workerZNodeParentPath='" + getZNodeParentPath(ZKNodeType.WORKER) + '\'' + diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClient.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClient.java new file mode 100644 index 0000000000..c08da0ef72 --- /dev/null +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClient.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.service.zk; + +import org.apache.commons.lang.StringUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.framework.api.ACLProvider; +import org.apache.curator.framework.state.ConnectionState; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.data.ACL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull; + +/** + * Shared Curator zookeeper client + */ +@Component +public class CuratorZookeeperClient implements InitializingBean { + private final Logger logger = LoggerFactory.getLogger(CuratorZookeeperClient.class); + + @Autowired + private ZookeeperConfig zookeeperConfig; + + private CuratorFramework zkClient; + + + @Override + public void afterPropertiesSet() throws Exception { + this.zkClient = buildClient(); + initStateLister(); + } + + private CuratorFramework buildClient() { + logger.info("zookeeper registry center init, server lists is: {}.", zookeeperConfig.getServerList()); + + CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder().ensembleProvider(new DefaultEnsembleProvider(checkNotNull(zookeeperConfig.getServerList(),"zookeeper quorum can't be null"))) + .retryPolicy(new ExponentialBackoffRetry(zookeeperConfig.getBaseSleepTimeMs(), zookeeperConfig.getMaxRetries(), zookeeperConfig.getMaxSleepMs())); + + //these has default value + if (0 != zookeeperConfig.getSessionTimeoutMs()) { + builder.sessionTimeoutMs(zookeeperConfig.getSessionTimeoutMs()); + } + if (0 != zookeeperConfig.getConnectionTimeoutMs()) { + builder.connectionTimeoutMs(zookeeperConfig.getConnectionTimeoutMs()); + } + if (StringUtils.isNotBlank(zookeeperConfig.getDigest())) { + builder.authorization("digest", zookeeperConfig.getDigest().getBytes(StandardCharsets.UTF_8)).aclProvider(new ACLProvider() { + + @Override + public List getDefaultAcl() { + return ZooDefs.Ids.CREATOR_ALL_ACL; + } + + @Override + public List getAclForPath(final String path) { + return ZooDefs.Ids.CREATOR_ALL_ACL; + } + }); + } + zkClient = builder.build(); + zkClient.start(); + try { + zkClient.blockUntilConnected(); + } catch (final Exception ex) { + throw new RuntimeException(ex); + } + return zkClient; + } + + public void initStateLister() { + checkNotNull(zkClient); + + zkClient.getConnectionStateListenable().addListener((client, newState) -> { + if(newState == ConnectionState.LOST){ + logger.error("connection lost from zookeeper"); + } else if(newState == ConnectionState.RECONNECTED){ + logger.info("reconnected to zookeeper"); + } else if(newState == ConnectionState.SUSPENDED){ + logger.warn("connection SUSPENDED to zookeeper"); + } + }); + } + + public ZookeeperConfig getZookeeperConfig() { + return zookeeperConfig; + } + + public void setZookeeperConfig(ZookeeperConfig zookeeperConfig) { + this.zookeeperConfig = zookeeperConfig; + } + + public CuratorFramework getZkClient() { + return zkClient; + } +} \ No newline at end of file diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperCachedOperator.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperCachedOperator.java index e71cb74e15..4fe941503e 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperCachedOperator.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperCachedOperator.java @@ -39,7 +39,7 @@ public class ZookeeperCachedOperator extends ZookeeperOperator { */ @Override protected void registerListener() { - treeCache = new TreeCache(zkClient, getZookeeperConfig().getDsRoot() + "/nodes"); + treeCache = new TreeCache(getZkClient(), getZookeeperConfig().getDsRoot() + "/nodes"); logger.info("add listener to zk path: {}", getZookeeperConfig().getDsRoot()); try { treeCache.start(); @@ -83,6 +83,7 @@ public class ZookeeperCachedOperator extends ZookeeperOperator { try { Thread.sleep(500); } catch (InterruptedException ignore) { + Thread.currentThread().interrupt(); } super.close(); } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperOperator.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperOperator.java index ba3a3bfecb..7aeb7289c4 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperOperator.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperOperator.java @@ -50,14 +50,10 @@ public class ZookeeperOperator implements InitializingBean { private final Logger logger = LoggerFactory.getLogger(ZookeeperOperator.class); @Autowired - private ZookeeperConfig zookeeperConfig; - - protected CuratorFramework zkClient; + private CuratorZookeeperClient zookeeperClient; @Override public void afterPropertiesSet() throws Exception { - this.zkClient = buildClient(); - initStateLister(); registerListener(); } @@ -66,62 +62,9 @@ public class ZookeeperOperator implements InitializingBean { */ protected void registerListener(){} - public void initStateLister() { - checkNotNull(zkClient); - - zkClient.getConnectionStateListenable().addListener((client, newState) -> { - if(newState == ConnectionState.LOST){ - logger.error("connection lost from zookeeper"); - } else if(newState == ConnectionState.RECONNECTED){ - logger.info("reconnected to zookeeper"); - } else if(newState == ConnectionState.SUSPENDED){ - logger.warn("connection SUSPENDED to zookeeper"); - } - }); - } - - private CuratorFramework buildClient() { - logger.info("zookeeper registry center init, server lists is: {}.", zookeeperConfig.getServerList()); - - CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder().ensembleProvider(new DefaultEnsembleProvider(checkNotNull(zookeeperConfig.getServerList(),"zookeeper quorum can't be null"))) - .retryPolicy(new ExponentialBackoffRetry(zookeeperConfig.getBaseSleepTimeMs(), zookeeperConfig.getMaxRetries(), zookeeperConfig.getMaxSleepMs())); - - //these has default value - if (0 != zookeeperConfig.getSessionTimeoutMs()) { - builder.sessionTimeoutMs(zookeeperConfig.getSessionTimeoutMs()); - } - if (0 != zookeeperConfig.getConnectionTimeoutMs()) { - builder.connectionTimeoutMs(zookeeperConfig.getConnectionTimeoutMs()); - } - if (StringUtils.isNotBlank(zookeeperConfig.getDigest())) { - builder.authorization("digest", zookeeperConfig.getDigest().getBytes(StandardCharsets.UTF_8)).aclProvider(new ACLProvider() { - - @Override - public List getDefaultAcl() { - return ZooDefs.Ids.CREATOR_ALL_ACL; - } - - @Override - public List getAclForPath(final String path) { - return ZooDefs.Ids.CREATOR_ALL_ACL; - } - }); - } - zkClient = builder.build(); - zkClient.start(); - try { - if (!zkClient.blockUntilConnected(zookeeperConfig.getMaxWaitTime(), TimeUnit.MILLISECONDS)) { - throw new IllegalStateException("Connect zookeeper expire max wait time"); - } - } catch (final Exception ex) { - throw new RuntimeException(ex); - } - return zkClient; - } - public String get(final String key) { try { - return new String(zkClient.getData().forPath(key), StandardCharsets.UTF_8); + return new String(zookeeperClient.getZkClient().getData().forPath(key), StandardCharsets.UTF_8); } catch (Exception ex) { logger.error("get key : {}", key, ex); } @@ -131,7 +74,7 @@ public class ZookeeperOperator implements InitializingBean { public List getChildrenKeys(final String key) { List values; try { - values = zkClient.getChildren().forPath(key); + values = zookeeperClient.getZkClient().getChildren().forPath(key); return values; } catch (InterruptedException ex) { logger.error("getChildrenKeys key : {} InterruptedException", key); @@ -145,7 +88,7 @@ public class ZookeeperOperator implements InitializingBean { public boolean hasChildren(final String key){ Stat stat ; try { - stat = zkClient.checkExists().forPath(key); + stat = zookeeperClient.getZkClient().checkExists().forPath(key); return stat.getNumChildren() >= 1; } catch (Exception ex) { throw new IllegalStateException(ex); @@ -154,7 +97,7 @@ public class ZookeeperOperator implements InitializingBean { public boolean isExisted(final String key) { try { - return zkClient.checkExists().forPath(key) != null; + return zookeeperClient.getZkClient().checkExists().forPath(key) != null; } catch (Exception ex) { logger.error("isExisted key : {}", key, ex); } @@ -164,7 +107,7 @@ public class ZookeeperOperator implements InitializingBean { public void persist(final String key, final String value) { try { if (!isExisted(key)) { - zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(key, value.getBytes(StandardCharsets.UTF_8)); + zookeeperClient.getZkClient().create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(key, value.getBytes(StandardCharsets.UTF_8)); } else { update(key, value); } @@ -176,9 +119,9 @@ public class ZookeeperOperator implements InitializingBean { public void update(final String key, final String value) { try { - CuratorOp check = zkClient.transactionOp().check().forPath(key); - CuratorOp setData = zkClient.transactionOp().setData().forPath(key, value.getBytes(StandardCharsets.UTF_8)); - zkClient.transaction().forOperations(check, setData); + CuratorOp check = zookeeperClient.getZkClient().transactionOp().check().forPath(key); + CuratorOp setData = zookeeperClient.getZkClient().transactionOp().setData().forPath(key, value.getBytes(StandardCharsets.UTF_8)); + zookeeperClient.getZkClient().transaction().forOperations(check, setData); } catch (Exception ex) { logger.error("update key : {} , value : {}", key, value, ex); @@ -189,12 +132,12 @@ public class ZookeeperOperator implements InitializingBean { try { if (isExisted(key)) { try { - zkClient.delete().deletingChildrenIfNeeded().forPath(key); + zookeeperClient.getZkClient().delete().deletingChildrenIfNeeded().forPath(key); } catch (KeeperException.NoNodeException ignore) { //NOP } } - zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath(key, value.getBytes(StandardCharsets.UTF_8)); + zookeeperClient.getZkClient().create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath(key, value.getBytes(StandardCharsets.UTF_8)); } catch (final Exception ex) { logger.error("persistEphemeral key : {} , value : {}", key, value, ex); } @@ -206,7 +149,7 @@ public class ZookeeperOperator implements InitializingBean { persistEphemeral(key, value); } else { if (!isExisted(key)) { - zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath(key, value.getBytes(StandardCharsets.UTF_8)); + zookeeperClient.getZkClient().create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath(key, value.getBytes(StandardCharsets.UTF_8)); } } } catch (final Exception ex) { @@ -216,7 +159,7 @@ public class ZookeeperOperator implements InitializingBean { public void persistEphemeralSequential(final String key, String value) { try { - zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(key, value.getBytes(StandardCharsets.UTF_8)); + zookeeperClient.getZkClient().create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(key, value.getBytes(StandardCharsets.UTF_8)); } catch (final Exception ex) { logger.error("persistEphemeralSequential key : {}", key, ex); } @@ -225,7 +168,7 @@ public class ZookeeperOperator implements InitializingBean { public void remove(final String key) { try { if (isExisted(key)) { - zkClient.delete().deletingChildrenIfNeeded().forPath(key); + zookeeperClient.getZkClient().delete().deletingChildrenIfNeeded().forPath(key); } } catch (KeeperException.NoNodeException ignore) { //NOP @@ -235,14 +178,14 @@ public class ZookeeperOperator implements InitializingBean { } public CuratorFramework getZkClient() { - return zkClient; + return zookeeperClient.getZkClient(); } public ZookeeperConfig getZookeeperConfig() { - return zookeeperConfig; + return zookeeperClient.getZookeeperConfig(); } public void close() { - CloseableUtils.closeQuietly(zkClient); + CloseableUtils.closeQuietly(zookeeperClient.getZkClient()); } } \ No newline at end of file diff --git a/dolphinscheduler-service/src/main/resources/logback-zookeeper.xml b/dolphinscheduler-service/src/main/resources/logback-zookeeper.xml new file mode 100644 index 0000000000..34a15a7a5b --- /dev/null +++ b/dolphinscheduler-service/src/main/resources/logback-zookeeper.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + + + + ${log.base}/dolphinscheduler-zookeeper.log + + ${log.base}/dolphinscheduler-alert.%d{yyyy-MM-dd_HH}.%i.log + 20 + 64MB + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + + + + + + + + diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClientTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClientTest.java new file mode 100644 index 0000000000..c0297799ea --- /dev/null +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClientTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.service.zk; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +public class CuratorZookeeperClientTest { + private static ZKServer zkServer; + + @Before + public void before() throws IOException { + new Thread(() -> { + if (zkServer == null) { + zkServer = new ZKServer(); + } + zkServer.startLocalZkServer(2185); + }).start(); + } + + @After + public void after() { + if (zkServer != null) { + zkServer.stop(); + } + } + + @Test + public void testAfterPropertiesSet() throws Exception { + TimeUnit.SECONDS.sleep(10); + CuratorZookeeperClient zookeeperClient = new CuratorZookeeperClient(); + ZookeeperConfig zookeeperConfig = new ZookeeperConfig(); + zookeeperConfig.setServerList("127.0.0.1:2185"); + zookeeperConfig.setBaseSleepTimeMs(100); + zookeeperConfig.setMaxSleepMs(30000); + zookeeperConfig.setMaxRetries(10); + zookeeperConfig.setSessionTimeoutMs(60000); + zookeeperConfig.setConnectionTimeoutMs(30000); + zookeeperConfig.setDigest(" "); + zookeeperConfig.setDsRoot("/dolphinscheduler"); + zookeeperConfig.setMaxWaitTime(30000); + zookeeperClient.setZookeeperConfig(zookeeperConfig); + System.out.println("start"); + zookeeperClient.afterPropertiesSet(); + System.out.println("end"); + Assert.assertNotNull(zookeeperClient.getZkClient()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-ui/build/config.js b/dolphinscheduler-ui/build/config.js index 53a421d9a0..2800115d80 100644 --- a/dolphinscheduler-ui/build/config.js +++ b/dolphinscheduler-ui/build/config.js @@ -116,7 +116,6 @@ const pages = glob.sync(['*/!(_*).html'], { cwd: viewDir }).map(p => { minify: minifierConfig }) }) - const baseConfig = { entry: jsEntry, output: { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js index 56d0168893..a4960f7ac5 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js @@ -153,7 +153,7 @@ const tasksState = { icoUnicode: 'ans-icon-dot-circle', isSpin: false }, - RUNNING_EXEUTION: { + RUNNING_EXECUTION: { id: 1, desc: `${i18n.$t('Executing')}`, color: '#0097e0', @@ -243,6 +243,10 @@ const tasksType = { desc: 'SHELL', color: '#646464' }, + WATERDROP: { + desc: 'WATERDROP', + color: '#646465' + }, SUB_PROCESS: { desc: 'SUB_PROCESS', color: '#0097e0' diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss index a3fcfb4f14..88f2d11c8f 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss @@ -74,6 +74,9 @@ .icos-SHELL { background: url("../img/toolbar_SHELL.png") no-repeat 50% 50%; } + .icos-WATERDROP { + background: url("../img/toolbar_WATERDROP.png") no-repeat 50% 50%; + } .icos-SUB_PROCESS { background: url("../img/toolbar_SUB_PROCESS.png") no-repeat 50% 50%; } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index d3df6adb77..6f07f97f02 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -38,7 +38,7 @@ type="text" v-model="name" :disabled="isDetails" - :placeholder="$t('Please enter name(required)')" + :placeholder="$t('Please enter name (required)')" maxlength="100" @on-blur="_verifName()" autocomplete="off"> @@ -162,6 +162,14 @@ ref="SHELL" :backfill-item="backfillItem"> + + + - + @@ -79,7 +79,7 @@ // btn loading this.isLoading = true this.$emit('dependItemListEvent', _.concat(this.dependItemList, this._rtNewParams())) - + // remove tooltip this._removeTip() }, @@ -223,4 +223,4 @@ } } } - \ No newline at end of file + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/waterdrop.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/waterdrop.vue new file mode 100644 index 0000000000..878d0f2834 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/waterdrop.vue @@ -0,0 +1,467 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js index 08682a4105..03c3d2dcbf 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js @@ -799,7 +799,7 @@ JSP.prototype.jspBackfill = function ({ connects, locations, largeJson }) { type: 'basic', paintStyle: { strokeWidth: 2, stroke: '#4caf50' }, HoverPaintStyle: {stroke: '#ccc', strokeWidth: 3}, - overlays:[["Label", { label: i18n.$t('success'), location:0.5, id:"label"} ]] + overlays:[["Label", { label: labels} ]] }) } else if($(`#${sourceId}`).attr('data-tasks-type') === 'CONDITIONS' && $(`#${sourceId}`).attr('data-failednode') === $(`#${targetId}`).find('.name-p').text()) { this.JspInstance.connect({ @@ -808,7 +808,7 @@ JSP.prototype.jspBackfill = function ({ connects, locations, largeJson }) { type: 'basic', paintStyle: { strokeWidth: 2, stroke: '#252d39' }, HoverPaintStyle: {stroke: '#ccc', strokeWidth: 3}, - overlays:[["Label", { label: i18n.$t('failed'), location:0.5, id:"label"} ]] + overlays:[["Label", { label: labels} ]] }) } else { this.JspInstance.connect({ diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue index b9610cbc7b..b881a80875 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue @@ -26,7 +26,7 @@ type="text" v-model="name" :disabled="router.history.current.name === 'projects-instance-details'" - :placeholder="$t('Please enter name(required)')"> + :placeholder="$t('Please enter name (required)')"> @@ -169,17 +169,12 @@ this.$emit('onUdp') } - // Edit => direct storage - if (this.store.state.dag.name) { + // verify that the name exists + this.store.dispatch('dag/verifDAGName', this.name).then(res => { _verif() - } else { - // New First verify that the name exists - this.store.dispatch('dag/verifDAGName', this.name).then(res => { - _verif() - }).catch(e => { - this.$message.error(e.msg || '') - }) - } + }).catch(e => { + this.$message.error(e.msg || '') + }) }, /** * Close the popup diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toolbar_WATERDROP.png b/dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toolbar_WATERDROP.png new file mode 100644 index 0000000000..8a09c569b1 Binary files /dev/null and b/dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toolbar_WATERDROP.png differ diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/index.vue index 13aa0536f5..cbf4a728e6 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/index.vue @@ -17,7 +17,9 @@ + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/kinship/_source/graphGridOption.js b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/kinship/_source/graphGridOption.js new file mode 100644 index 0000000000..117a177651 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/kinship/_source/graphGridOption.js @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import _ from 'lodash'; +import i18n from '@/module/i18n/index.js' + +const getCategory = (categoryDic, { workFlowPublishStatus, schedulePublishStatus, id }, sourceWorkFlowId) => { + if (id === sourceWorkFlowId) return categoryDic['active'] + switch (true) { + case workFlowPublishStatus === '0': + return categoryDic['0']; + case workFlowPublishStatus === '1' && schedulePublishStatus === '0': + return categoryDic['10']; + case workFlowPublishStatus === '1' && schedulePublishStatus === '1': + default: + return categoryDic['1']; + } +} + +export default function (locations, links, sourceWorkFlowId, isShowLabel) { + + const categoryDic = { + 'active': { color: '#2D8DF0', category: i18n.$t('KinshipStateActive')}, + '1': { color: '#00C800', category: i18n.$t('KinshipState1')}, + '0': { color: '#999999', category: i18n.$t('KinshipState0')}, + '10': { color: '#FF8F05', category: i18n.$t('KinshipState10')}, + } + const newData = _.map(locations, (item) => { + const { color, category } = getCategory(categoryDic, item, sourceWorkFlowId) + return { + ...item, + emphasis: { + itemStyle: { + color + }, + }, + category + } + }); + + const categories = [ + { name: categoryDic.active.category}, + { name: categoryDic['1'].category}, + { name: categoryDic['0'].category}, + { name: categoryDic['10'].category}, + ] + let option = { + tooltip: { + trigger: 'item', + triggerOn: 'mousemove', + backgroundColor: '#2D303A', + padding: [8, 12], + formatter: (params) => { + if (!params.data.name) return ''; + const { name, scheduleStartTime, scheduleEndTime, crontab, workFlowPublishStatus, schedulePublishStatus } = params.data; + const str = ` + 工作流名字：${name}
+ 调度开始时间：${scheduleStartTime}
+ 调度结束时间：${scheduleEndTime}
+ crontab表达式：${crontab}
+ 工作流发布状态：${workFlowPublishStatus}
+ 调度发布状态：${schedulePublishStatus}
+ ` + return str; + }, + color: '#2D303A', + textStyle: { + rich: { + a: { + fontSize: 12, + color: '#2D303A', + lineHeight: 12, + align: 'left', + padding: [4, 4, 4, 4] + }, + } + }, + }, + color: [categoryDic.active.color, categoryDic['1'].color, categoryDic['0'].color, categoryDic['10'].color], + legend: [{ + orient: 'horizontal', + top: 6, + left: 6, + data: categories, + }], + series: [{ + type: 'graph', + layout: 'force', + nodeScaleRatio: 1.2, + draggable: true, + animation: false, + data: newData, + roam: true, + symbol: 'roundRect', + symbolSize: 70, + categories, + label: { + show: isShowLabel, + position: 'inside', + formatter: (params) => { + if (!params.data.name) return ''; + const str = params.data.name.split('_').map(item => `{a|${item}\n}`).join('') + return str; + }, + color: '#222222', + textStyle: { + rich: { + a: { + fontSize: 12, + color: '#222222', + lineHeight: 12, + align: 'left', + padding: [4, 4, 4, 4] + }, + } + } + }, + edgeSymbol: ['circle', 'arrow'], + edgeSymbolSize: [4, 12], + force: { + repulsion: 1000, + edgeLength: 300 + }, + links: links, + lineStyle: { + color: '#999999' + } + }] + }; + + return option +} diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/kinship/_source/img/dag_bg.png b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/kinship/_source/img/dag_bg.png new file mode 100644 index 0000000000..d9fa09817e Binary files /dev/null and b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/kinship/_source/img/dag_bg.png differ diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/kinship/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/kinship/index.vue new file mode 100644 index 0000000000..2472a69786 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/kinship/index.vue @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + diff --git a/dolphinscheduler-ui/src/js/conf/home/router/index.js b/dolphinscheduler-ui/src/js/conf/home/router/index.js index b4236f3685..b65586c433 100644 --- a/dolphinscheduler-ui/src/js/conf/home/router/index.js +++ b/dolphinscheduler-ui/src/js/conf/home/router/index.js @@ -57,6 +57,14 @@ const router = new Router({ title: `${i18n.$t('Project Home')}` } }, + { + path: '/projects/kinship', + name: 'projects-kinship', + component: resolve => require(['../pages/projects/pages/kinship/index'], resolve), + meta: { + title: `${i18n.$t('Kinship')}` + } + }, { path: '/projects/list', name: 'projects-list', diff --git a/dolphinscheduler-ui/src/js/conf/home/store/index.js b/dolphinscheduler-ui/src/js/conf/home/store/index.js index 5ca2187479..a7e5824ff0 100644 --- a/dolphinscheduler-ui/src/js/conf/home/store/index.js +++ b/dolphinscheduler-ui/src/js/conf/home/store/index.js @@ -17,6 +17,7 @@ import Vue from 'vue' import Vuex from 'vuex' import dag from './dag' +import kinship from './kinship' import projects from './projects' import resource from './resource' import security from './security' @@ -28,6 +29,7 @@ export default new Vuex.Store({ modules: { dag, projects, + kinship, resource, security, datasource, diff --git a/dolphinscheduler-ui/src/js/conf/home/store/kinship/actions.js b/dolphinscheduler-ui/src/js/conf/home/store/kinship/actions.js new file mode 100644 index 0000000000..f87a57d17f --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/store/kinship/actions.js @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import _ from 'lodash' +import io from '@/module/io' +import localStore from '@/module/util/localStorage' + +export default { + /** + * Get workFlow DAG + */ + getWorkFlowList ({ state }, payload) { + const projectId = localStore.getItem('projectId'); + return new Promise((resolve, reject) => { + const url = `lineages/${projectId}/list-name`; + io.get(url, { + searchVal: payload, + }, res => { + const workList = []; + if (res.data) { + _.map(res.data, (item) => { + workList.push({ + id: `${item.workFlowId}`, + name: item.workFlowName, + }) + }) + } + state.workList = workList /* JSON.parse(connects) */ + resolve(res.data) + }).catch(res => { + reject(res) + }) + }) + }, + /** + * Get workFlow DAG + */ + getWorkFlowDAG ({ state }, payload) { + const projectId = localStore.getItem('projectId'); + return new Promise((resolve, reject) => { + const url = `lineages/${projectId}/list-ids`; + io.get(url, { + ids: payload, + }, res => { + let locations = []; + let connects = []; + if (res.data.workFlowList) { + locations = _.uniqBy(res.data.workFlowList, 'workFlowId').map((item) => ({ + id: `${item.workFlowId}`, + name: item.workFlowName, + workFlowPublishStatus: item.workFlowPublishStatus, + scheduleStartTime: item.scheduleStartTime, + scheduleEndTime: item.scheduleEndTime, + crontab: item.crontab, + schedulePublishStatus: item.schedulePublishStatus + })) + } + if (res.data.workFlowRelationList) { + connects = _.map(res.data.workFlowRelationList, (item) => ({ + source: `${item.sourceWorkFlowId}`, // should be string, or connects will not show by echarts + target: `${item.targetWorkFlowId}`, // should be string, or connects will not show by echarts + })) + } + state.sourceWorkFlowId = payload || ''; + // locations + state.locations = locations /* JSON.parse(locations) */ + // connects + state.connects = connects /* JSON.parse(connects) */ + resolve(res.data) + }).catch(res => { + reject(res) + }) + }) + }, +} diff --git a/dolphinscheduler-ui/src/js/conf/home/store/kinship/getters.js b/dolphinscheduler-ui/src/js/conf/home/store/kinship/getters.js new file mode 100644 index 0000000000..e84c864c3c --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/store/kinship/getters.js @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export default { +} diff --git a/dolphinscheduler-ui/src/js/conf/home/store/kinship/index.js b/dolphinscheduler-ui/src/js/conf/home/store/kinship/index.js new file mode 100644 index 0000000000..24abd1ccaa --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/store/kinship/index.js @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import actions from './actions' +import getters from './getters' +import mutations from './mutations' +import state from './state' + +export default { + strict: true, + namespaced: true, + state, + getters, + mutations, + actions +} diff --git a/dolphinscheduler-ui/src/js/conf/home/store/kinship/mutations.js b/dolphinscheduler-ui/src/js/conf/home/store/kinship/mutations.js new file mode 100644 index 0000000000..2d71cd834f --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/store/kinship/mutations.js @@ -0,0 +1,17 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export default {} diff --git a/dolphinscheduler-ui/src/js/conf/home/store/kinship/state.js b/dolphinscheduler-ui/src/js/conf/home/store/kinship/state.js new file mode 100644 index 0000000000..6f22b86477 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/store/kinship/state.js @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export default { + sourceWorkFlowId: '', + workList: [], + locations: [], + connects: [] +} diff --git a/dolphinscheduler-ui/src/js/module/components/secondaryMenu/_source/menu.js b/dolphinscheduler-ui/src/js/module/components/secondaryMenu/_source/menu.js index 6487c2faef..56834a294e 100644 --- a/dolphinscheduler-ui/src/js/module/components/secondaryMenu/_source/menu.js +++ b/dolphinscheduler-ui/src/js/module/components/secondaryMenu/_source/menu.js @@ -31,8 +31,17 @@ const menu = { children: [] }, { - name: `${i18n.$t('Process')}`, + name: `${i18n.$t('Kinship')}`, id: 1, + path: 'projects-kinship', + isOpen: true, + disabled: true, + icon: 'ans-icon-node', + children: [] + }, + { + name: `${i18n.$t('Process')}`, + id: 2, path: '', isOpen: true, disabled: true, diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index 2cca526bdd..f5e579c088 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -34,7 +34,6 @@ export default { 'View log': 'View log', 'Enter this child node': 'Enter this child node', 'Node name': 'Node name', - 'Please enter name(required)': 'Please enter name(required)', 'Run flag': 'Run flag', Normal: 'Normal', 'Prohibition execution': 'Prohibition execution', @@ -546,8 +545,8 @@ export default { '0 means unlimited by count': '0 means unlimited', 'Modify User': 'Modify User', 'Whether directory': 'Whether directory', - Yes: 'Yes', - No: 'No', + 'Yes': 'Yes', + 'No': 'No', 'Hadoop Custom Params': 'Hadoop Params', 'Sqoop Advanced Parameters': 'Sqoop Params', 'Sqoop Job Name': 'Job Name', @@ -606,6 +605,13 @@ export default { 'Successful branch flow and failed branch flow are required': 'conditions node Successful and failed branch flow are required', 'Unauthorized or deleted resources': 'Unauthorized or deleted resources', 'Please delete all non-existent resources': 'Please delete all non-existent resources', + 'Kinship': 'Workflow relationship', + 'Reset': 'Reset', + 'KinshipStateActive': 'Active', + 'KinshipState1': 'Online', + 'KinshipState0': 'Workflow is not online', + 'KinshipState10': 'Scheduling is not online', + 'Dag label display control': 'Dag label display control', 'Enable': 'Enable', 'Timeout Settings': 'Timeout Settings', 'Connect Timeout':'Connect Timeout', @@ -617,6 +623,9 @@ export default { 'The Worker group no longer exists, please select the correct Worker group!': 'The Worker group no longer exists, please select the correct Worker group!', 'Please confirm whether the workflow has been saved before downloading': 'Please confirm whether the workflow has been saved before downloading', 'User name length is between 3 and 39': 'User name length is between 3 and 39', + 'Please Enter Url': 'Please Enter Url eg. 127.0.0.1:7077', + 'Master': 'Master', + 'Please select the waterdrop resources':'Please select the waterdrop resources', zkDirectory: 'zkDirectory', 'Directory detail': 'Directory detail', 'Connection name': 'Connection name', diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index 425f46a147..8d524144e1 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -34,7 +34,7 @@ export default { 'View log': '查看日志', 'Enter this child node': '进入该子节点', 'Node name': '节点名称', - 'Please enter name(required)': '请输入名称(必填)', + 'Please enter name (required)': '请输入名称(必填)', 'Run flag': '运行标志', Normal: '正常', 'Prohibition execution': '禁止执行', @@ -605,6 +605,13 @@ export default { 'Successful branch flow and failed branch flow are required': 'conditions节点成功和失败分支流转必填', 'Unauthorized or deleted resources': '未授权或已删除资源', 'Please delete all non-existent resources': '请删除所有未授权或已删除资源', + 'Kinship': '工作流关系', + 'Reset': '重置', + 'KinshipStateActive': '当前选择', + 'KinshipState1': '已上线', + 'KinshipState0': '工作流未上线', + 'KinshipState10': '调度未上线', + 'Dag label display control': 'Dag节点名称显隐', 'Enable': '启用', 'Disable': '停用', 'The Worker group no longer exists, please select the correct Worker group!': '该Worker分组已经不存在，请选择正确的Worker分组！', @@ -616,6 +623,9 @@ export default { 'Connect timeout be a positive integer': '连接超时必须为数字', 'Socket Timeout be a positive integer': 'Socket超时必须为数字', 'ms':'毫秒', + 'Please Enter Url': '请直接填写地址,例如:127.0.0.1:7077', + 'Master': 'Master', + 'Please select the waterdrop resources':'请选择waterdrop配置文件', zkDirectory: 'zk注册目录', 'Directory detail': '查看目录详情', 'Connection name': '连线名', diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProcessInstanceLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProcessInstanceLocator.java index 1af9d56274..445ea3ee78 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProcessInstanceLocator.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProcessInstanceLocator.java @@ -21,7 +21,7 @@ import org.openqa.selenium.By; public class ProcessInstanceLocator { // jump Process Instance page //click Process Instance name - public static final By CLICK_PROCESS_INSTANCE_NAME = By.xpath("//div[3]/div/ul/li[2]"); + public static final By CLICK_PROCESS_INSTANCE_NAME = By.xpath("//div[4]/div/ul/li[2]"); // click rerun button public static final By CLICK_RERUN_BUTTON = By.xpath("//tr[2]/td[14]/div[1]/button[2]"); diff --git a/e2e/testng.xml b/e2e/testng.xml index 499fe113bc..7e5dc5228b 100644 --- a/e2e/testng.xml +++ b/e2e/testng.xml @@ -78,7 +78,7 @@ - + diff --git a/install.sh b/install.sh old mode 100644 new mode 100755 index a74fe67531..ae16a1d199 --- a/install.sh +++ b/install.sh @@ -53,6 +53,7 @@ sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${k sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common.properties sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common.properties sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/zookeeper.properties +sed -i ${txt} "s\#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/zookeeper.properties sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application-api.properties sed -i ${txt} "s#mail.server.host.*#mail.server.host=${mailServerHost}#g" conf/alert.properties sed -i ${txt} "s#mail.server.port.*#mail.server.port=${mailServerPort}#g" conf/alert.properties @@ -96,4 +97,8 @@ sh ${workDir}/script/remove-zk-node.sh $zkRoot # 6.startup echo "6.startup" -sh ${workDir}/script/start-all.sh \ No newline at end of file +sh ${workDir}/script/start-all.sh + +# 7.query status +echo "7.query status" +sh ${workDir}/script/status-all.sh diff --git a/pom.xml b/pom.xml index 4103c75ddb..b469f38c1d 100644 --- a/pom.xml +++ b/pom.xml @@ -552,6 +552,7 @@ apache-dolphinscheduler-incubating-${project.version} + @@ -733,7 +734,9 @@ **/api/service/UserAlertGroupServiceTest.java **/api/service/UsersServiceTest.java **/api/service/WorkerGroupServiceTest.java + **/api/service/WorkFlowLineageServiceTest.java **/api/controller/ProcessDefinitionControllerTest.java + **/api/controller/WorkFlowLineageControllerTest.java **/api/utils/exportprocess/DataSourceParamTest.java **/api/utils/exportprocess/DependentParamTest.java **/api/utils/CheckUtilsTest.java @@ -798,7 +801,7 @@ **/server/log/TaskLogFilterTest.java **/server/log/WorkerLogFilterTest.java - + **/server/master/consumer/TaskPriorityQueueConsumerTest.java **/server/master/runner/MasterTaskExecThreadTest.java **/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java @@ -811,14 +814,14 @@ **/server/master/MasterExecThreadTest.java **/server/master/ParamsTest.java - + **/server/register/ZookeeperNodeManagerTest.java **/server/utils/DataxUtilsTest.java **/server/utils/ExecutionContextTestUtils.java **/server/utils/ParamUtilsTest.java **/server/utils/ProcessUtilsTest.java **/server/utils/SparkArgsUtilsTest.java - + **/server/worker/processor/TaskCallbackServiceTest.java **/server/worker/registry/WorkerRegistryTest.java **/server/worker/shell/ShellCommandExecutorTest.java **/server/worker/sql/SqlExecutorTest.java @@ -832,6 +835,7 @@ **/service/quartz/cron/CronUtilsTest.java **/service/zk/DefaultEnsembleProviderTest.java **/service/zk/ZKServerTest.java + **/service/zk/CuratorZookeeperClientTest.java **/service/queue/TaskUpdateQueueTest.java **/dao/mapper/DataSourceUserMapperTest.java diff --git a/repository/dolphinscheduler/dolphinscheduler-ui/.gitignore b/repository/dolphinscheduler/dolphinscheduler-ui/.gitignore new file mode 100644 index 0000000000..27c1333c66 --- /dev/null +++ b/repository/dolphinscheduler/dolphinscheduler-ui/.gitignore @@ -0,0 +1,6 @@ +.idea +.settings +package-lock.json +.classpath +.project +node_modules \ No newline at end of file diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index 4ffd7fe4c2..8c96764336 100644 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -46,7 +46,7 @@ export DOLPHINSCHEDULER_LOG_DIR=$DOLPHINSCHEDULER_HOME/logs export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/* -export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-"-server -Xmx16g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"} +export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-"-server -Xmx16g -Xms1g -Xss512k -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=10m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"} export STOP_TIMEOUT=5 if [ ! -d "$DOLPHINSCHEDULER_LOG_DIR" ]; then @@ -72,6 +72,10 @@ elif [ "$command" = "alert-server" ]; then CLASS=org.apache.dolphinscheduler.alert.AlertServer elif [ "$command" = "logger-server" ]; then CLASS=org.apache.dolphinscheduler.server.log.LoggerServer +elif [ "$command" = "zookeeper-server" ]; then + #note: this command just for getting a quick experience，not recommended for production. this operation will start a standalone zookeeper server + LOG_FILE="-Dlogback.configurationFile=classpath:logback-zookeeper.xml" + CLASS=org.apache.dolphinscheduler.service.zk.ZKServer else echo "Error: No command named \`$command' was found." exit 1 @@ -125,4 +129,4 @@ case $startStop in esac -echo "End $startStop $command." \ No newline at end of file +echo "End $startStop $command." diff --git a/script/remove-zk-node.sh b/script/remove-zk-node.sh old mode 100644 new mode 100755 index 1fff1d5597..1129ca6a02 --- a/script/remove-zk-node.sh +++ b/script/remove-zk-node.sh @@ -31,18 +31,20 @@ BIN_DIR=`dirname $0` BIN_DIR=`cd "$BIN_DIR"; pwd` DOLPHINSCHEDULER_HOME=$BIN_DIR/.. +source ${BIN_DIR}/../conf/config/install_config.conf + export JAVA_HOME=$JAVA_HOME export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/* -export DOLPHINSCHEDULER_OPTS="-server -Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" +export DOLPHINSCHEDULER_OPTS="-Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 " export STOP_TIMEOUT=5 -CLASS=org.apache.dolphinscheduler.server.utils.RemoveZKNode +CLASS=org.apache.zookeeper.ZooKeeperMain -exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS $rootNode" +exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS -server $zkQuorum rmr $rootNode" cd $DOLPHINSCHEDULER_HOME $JAVA_HOME/bin/java $exec_command diff --git a/script/status-all.sh b/script/status-all.sh new file mode 100644 index 0000000000..64dfaa46b8 --- /dev/null +++ b/script/status-all.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +workDir=`dirname $0` +workDir=`cd ${workDir};pwd` +source $workDir/../conf/config/install_config.conf + +echo -e '\n' +echo "====================== dolphinscheduler install config=============================" +echo -e "1.dolphinscheduler server node install hosts:[ \033[1;32m ${ips} \033[0m ]" +echo -e "2.master server node install hosts:[ \033[1;32m ${masters} \033[0m ]" +echo -e "3.worker server node install hosts:[ \033[1;32m ${workers} \033[0m ]" +echo -e "4.alert server node install hosts:[ \033[1;32m ${alertServer} \033[0m ]" +echo -e "5.api server node install hosts:[ \033[1;32m ${apiServers} \033[0m ]" + +echo -e '\n' + + +ipsHost=(${ips//,/ }) +for ip in ${ipsHost[@]} +do + echo -e "====================== [ \033[1;32m ${ip} \033[0m ] node all servers ==========================" + ssh -p $sshPort $ip "jps" + echo -e '\n' +done + diff --git a/sql/create/release-1.0.0_schema/mysql/dolphinscheduler_dml.sql b/sql/create/release-1.0.0_schema/mysql/dolphinscheduler_dml.sql index e8c797b1a9..fd65d3091e 100644 --- a/sql/create/release-1.0.0_schema/mysql/dolphinscheduler_dml.sql +++ b/sql/create/release-1.0.0_schema/mysql/dolphinscheduler_dml.sql @@ -16,7 +16,7 @@ */ -- Records of t_escheduler_user,user : admin , password : dolphinscheduler123 -INSERT INTO `t_escheduler_user` VALUES ('1', 'admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', 'xx', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22'); +INSERT INTO `t_escheduler_user` VALUES ('1', 'admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', '', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22'); INSERT INTO `t_escheduler_alertgroup` VALUES (1, 'default admin warning group', '0', 'default admin warning group','2018-11-29 10:20:39', '2018-11-29 10:20:39'); INSERT INTO `t_escheduler_relation_user_alertgroup` VALUES ('1', '1', '1', '2018-11-29 10:22:33', '2018-11-29 10:22:33'); diff --git a/sql/create/release-1.2.0_schema/postgresql/dolphinscheduler_dml.sql b/sql/create/release-1.2.0_schema/postgresql/dolphinscheduler_dml.sql index 4b83d78fea..d075cab9bf 100644 --- a/sql/create/release-1.2.0_schema/postgresql/dolphinscheduler_dml.sql +++ b/sql/create/release-1.2.0_schema/postgresql/dolphinscheduler_dml.sql @@ -16,7 +16,7 @@ */ -- Records of t_ds_user，user : admin , password : dolphinscheduler123 -INSERT INTO t_ds_user(user_name,user_password,user_type,email,phone,tenant_id,create_time,update_time) VALUES ('admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', 'xx', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22'); +INSERT INTO t_ds_user(user_name,user_password,user_type,email,phone,tenant_id,create_time,update_time) VALUES ('admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', '', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22'); -- Records of t_ds_alertgroup，dolphinscheduler warning group INSERT INTO t_ds_alertgroup(group_name,group_type,description,create_time,update_time) VALUES ('dolphinscheduler warning group', '0', 'dolphinscheduler warning group','2018-11-29 10:20:39', '2018-11-29 10:20:39'); diff --git a/sql/dolphinscheduler-postgre.sql b/sql/dolphinscheduler-postgre.sql index 1b27c7b733..4a947d134d 100644 --- a/sql/dolphinscheduler-postgre.sql +++ b/sql/dolphinscheduler-postgre.sql @@ -752,7 +752,7 @@ ALTER TABLE t_ds_worker_server ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_worker_ -- Records of t_ds_user?user : admin , password : dolphinscheduler123 -INSERT INTO t_ds_user(user_name,user_password,user_type,email,phone,tenant_id,state,create_time,update_time) VALUES ('admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', 'xx', '0', 1, '2018-03-27 15:48:50', '2018-10-24 17:40:22'); +INSERT INTO t_ds_user(user_name,user_password,user_type,email,phone,tenant_id,state,create_time,update_time) VALUES ('admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', '', '0', 1, '2018-03-27 15:48:50', '2018-10-24 17:40:22'); -- Records of t_ds_alertgroup，dolphinscheduler warning group INSERT INTO t_ds_alertgroup(group_name,group_type,description,create_time,update_time) VALUES ('dolphinscheduler warning group', '0', 'dolphinscheduler warning group','2018-11-29 10:20:39', '2018-11-29 10:20:39'); diff --git a/sql/dolphinscheduler_mysql.sql b/sql/dolphinscheduler_mysql.sql index cac32b63e8..f4ad9a3cb8 100644 --- a/sql/dolphinscheduler_mysql.sql +++ b/sql/dolphinscheduler_mysql.sql @@ -815,4 +815,4 @@ INSERT INTO `t_ds_relation_user_alertgroup` VALUES ('1', '1', '1', '2018-11-29 1 -- ---------------------------- -- Records of t_ds_user -- ---------------------------- -INSERT INTO `t_ds_user` VALUES ('1', 'admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', 'xx', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22', null, 1); +INSERT INTO `t_ds_user` VALUES ('1', 'admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', '', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22', null, 1); diff --git a/style/intellij-java-code-style.xml b/style/intellij-java-code-style.xml new file mode 100644 index 0000000000..ae333b1c3d --- /dev/null +++ b/style/intellij-java-code-style.xml @@ -0,0 +1,82 @@ + + + + + + + + + + + + \ No newline at end of file