github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

[Improvement][service] Refactor service module to fix code smell (#4513) 

* chore: Refactor code to fix code smell in service module

* chore: Add licence header to new files

* chore: Fix comment from code review

* chore[service]: Reduce the number of custom runtime exception to one
pull/3/MERGE
Segun Ogundipe 4 years ago committed by GitHub
parent
afd201e6fa
commit
e083e28720
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 1095 additions and 987 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java
  2. 5
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java
  3. 52
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/exceptions/ServiceException.java
  4. 48
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java
  5. 307
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
  6. 15
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/ProcessScheduleJob.java
  7. 614
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java
  8. 302
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/AbstractCycle.java
  9. 611
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java
  10. 22
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClient.java
  11. 37
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java
  12. 20
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperCachedOperator.java
  13. 45
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperOperator.java

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java
Unescape View File

@ -61,4 +61,8 @@ public class StringUtils {
public static String trim(String str) { public static String trim(String str) {
return str == null ? null : str.trim(); return str == null ? null : str.trim();
} }
public static boolean equalsIgnoreCase(String str1, String str2) {
return str1 == null ? str2 == null : str1.equalsIgnoreCase(str2);
}
} }

5
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java
Unescape View File

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.service.bean; package org.apache.dolphinscheduler.service.bean;
import org.springframework.beans.BeansException; import org.springframework.beans.BeansException;
@ -31,9 +32,7 @@ public class SpringApplicationContext implements ApplicationContextAware {
SpringApplicationContext.applicationContext = applicationContext; SpringApplicationContext.applicationContext = applicationContext;
} }
public static <T> T getBean(Class<T> requiredType){ public static <T> T getBean(Class<T> requiredType) {
return applicationContext.getBean(requiredType); return applicationContext.getBean(requiredType);
} }
} }

52
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/exceptions/ServiceException.java
Unescape View File

@ -0,0 +1,52 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.exceptions;
/**
* Custom ZKServerException exception
*/
public class ServiceException extends RuntimeException {
/**
* Construct a new runtime exception with the error message
*
* @param errMsg Error message
*/
public ServiceException(String errMsg) {
super(errMsg);
}
/**
* Construct a new runtime exception with the cause
*
* @param cause cause
*/
public ServiceException(Throwable cause) {
super(cause);
}
/**
* Construct a new runtime exception with the detail message and cause
*
* @param errMsg message
* @param cause cause
*/
public ServiceException(String errMsg, Throwable cause) {
super(errMsg, cause);
}
}

48
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java
Unescape View File

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.service.permission; package org.apache.dolphinscheduler.service.permission;
import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.AuthorizationType;
@ -21,11 +22,13 @@ import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.service.exceptions.ServiceException;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import org.slf4j.Logger;
import java.util.List; import java.util.List;
import org.slf4j.Logger;
public class PermissionCheck<T> { public class PermissionCheck<T> {
/** /**
* logger * logger
@ -58,8 +61,9 @@ public class PermissionCheck<T> {
/** /**
* permission check * permission check
*
* @param authorizationType authorization type * @param authorizationType authorization type
* @param processService process dao * @param processService process dao
*/ */
public PermissionCheck(AuthorizationType authorizationType, ProcessService processService) { public PermissionCheck(AuthorizationType authorizationType, ProcessService processService) {
this.authorizationType = authorizationType; this.authorizationType = authorizationType;
@ -68,10 +72,6 @@ public class PermissionCheck<T> {
/** /**
* permission check * permission check
* @param authorizationType
* @param processService
* @param needChecks
* @param userId
*/ */
public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, T[] needChecks, int userId) { public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, T[] needChecks, int userId) {
this.authorizationType = authorizationType; this.authorizationType = authorizationType;
@ -82,11 +82,6 @@ public class PermissionCheck<T> {
/** /**
* permission check * permission check
* @param authorizationType
* @param processService
* @param needChecks
* @param userId
* @param logger
*/ */
public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, T[] needChecks, int userId, Logger logger) { public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, T[] needChecks, int userId, Logger logger) {
this.authorizationType = authorizationType; this.authorizationType = authorizationType;
@ -98,13 +93,8 @@ public class PermissionCheck<T> {
/** /**
* permission check * permission check
* @param logger
* @param authorizationType
* @param processService
* @param resourceList
* @param userId
*/ */
public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, List<ResourceInfo> resourceList, int userId,Logger logger) { public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, List<ResourceInfo> resourceList, int userId, Logger logger) {
this.authorizationType = authorizationType; this.authorizationType = authorizationType;
this.processService = processService; this.processService = processService;
this.resourceList = resourceList; this.resourceList = resourceList;
@ -154,9 +144,10 @@ public class PermissionCheck<T> {
/** /**
* has permission * has permission
*
* @return true if has permission * @return true if has permission
*/ */
public boolean hasPermission(){ public boolean hasPermission() {
try { try {
checkPermission(); checkPermission();
return true; return true;
@ -167,23 +158,24 @@ public class PermissionCheck<T> {
/** /**
* check permission * check permission
* @throws Exception exception *
* @throws ServiceException exception
*/ */
public void checkPermission() throws Exception{ public void checkPermission() throws ServiceException {
if(this.needChecks.length > 0){ if (this.needChecks.length > 0) {
// get user type in order to judge whether the user is admin // get user type in order to judge whether the user is admin
User user = processService.getUserById(userId); User user = processService.getUserById(userId);
if (user == null) { if (user == null) {
logger.error("user id {} didn't exist",userId); logger.error("user id {} doesn't exist", userId);
throw new RuntimeException(String.format("user %s didn't exist",userId)); throw new ServiceException(String.format("user %s doesn't exist", userId));
} }
if (user.getUserType() != UserType.ADMIN_USER){ if (user.getUserType() != UserType.ADMIN_USER) {
List<T> unauthorizedList = processService.listUnauthorized(userId,needChecks,authorizationType); List<T> unauthorizedList = processService.listUnauthorized(userId, needChecks, authorizationType);
// if exist unauthorized resource // if exist unauthorized resource
if(CollectionUtils.isNotEmpty(unauthorizedList)){ if (CollectionUtils.isNotEmpty(unauthorizedList)) {
logger.error("user {} didn't has permission of {}: {}", user.getUserName(), authorizationType.getDescp(),unauthorizedList); logger.error("user {} doesn't have permission of {}: {}", user.getUserName(), authorizationType.getDescp(), unauthorizedList);
throw new RuntimeException(String.format("user %s didn't has permission of %s %s", user.getUserName(), authorizationType.getDescp(), unauthorizedList.get(0))); throw new ServiceException(String.format("user %s doesn't have permission of %s %s", user.getUserName(), authorizationType.getDescp(), unauthorizedList.get(0)));
} }
} }
} }

307
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
Unescape View File

@ -84,6 +84,7 @@ import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Calendar; import java.util.Calendar;
import java.util.Date; import java.util.Date;
import java.util.EnumMap;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
@ -110,11 +111,11 @@ public class ProcessService {
private final Logger logger = LoggerFactory.getLogger(getClass()); private final Logger logger = LoggerFactory.getLogger(getClass());
private final int[] stateArray = new int[] {ExecutionStatus.SUBMITTED_SUCCESS.ordinal(), private final int[] stateArray = new int[]{ExecutionStatus.SUBMITTED_SUCCESS.ordinal(),
ExecutionStatus.RUNNING_EXECUTION.ordinal(), ExecutionStatus.RUNNING_EXECUTION.ordinal(),
ExecutionStatus.DELAY_EXECUTION.ordinal(), ExecutionStatus.DELAY_EXECUTION.ordinal(),
ExecutionStatus.READY_PAUSE.ordinal(), ExecutionStatus.READY_PAUSE.ordinal(),
ExecutionStatus.READY_STOP.ordinal()}; ExecutionStatus.READY_STOP.ordinal()};
@Autowired @Autowired
private UserMapper userMapper; private UserMapper userMapper;
@ -158,16 +159,16 @@ public class ProcessService {
/** /**
* handle Command (construct ProcessInstance from Command) , wrapped in transaction * handle Command (construct ProcessInstance from Command) , wrapped in transaction
* *
* @param logger logger * @param logger logger
* @param host host * @param host host
* @param validThreadNum validThreadNum * @param validThreadNum validThreadNum
* @param command found command * @param command found command
* @return process instance * @return process instance
*/ */
@Transactional(rollbackFor = Exception.class) @Transactional(rollbackFor = Exception.class)
public ProcessInstance handleCommand(Logger logger, String host, int validThreadNum, Command command) { public ProcessInstance handleCommand(Logger logger, String host, int validThreadNum, Command command) {
ProcessInstance processInstance = constructProcessInstance(command, host); ProcessInstance processInstance = constructProcessInstance(command, host);
//cannot construct process instance, return null; // cannot construct process instance, return null
if (processInstance == null) { if (processInstance == null) {
logger.error("scan command, command parameter is error: {}", command); logger.error("scan command, command parameter is error: {}", command);
moveToErrorCommand(command, "process instance is null"); moveToErrorCommand(command, "process instance is null");
@ -201,7 +202,7 @@ public class ProcessService {
/** /**
* set process waiting thread * set process waiting thread
* *
* @param command command * @param command command
* @param processInstance processInstance * @param processInstance processInstance
* @return process instance * @return process instance
*/ */
@ -219,7 +220,7 @@ public class ProcessService {
/** /**
* check thread num * check thread num
* *
* @param command command * @param command command
* @param validThreadNum validThreadNum * @param validThreadNum validThreadNum
* @return if thread is enough * @return if thread is enough
*/ */
@ -259,7 +260,7 @@ public class ProcessService {
*/ */
public Boolean verifyIsNeedCreateCommand(Command command) { public Boolean verifyIsNeedCreateCommand(Command command) {
Boolean isNeedCreate = true; Boolean isNeedCreate = true;
Map<CommandType, Integer> cmdTypeMap = new HashMap<CommandType, Integer>(); EnumMap<CommandType, Integer> cmdTypeMap = new EnumMap<>(CommandType.class);
cmdTypeMap.put(CommandType.REPEAT_RUNNING, 1); cmdTypeMap.put(CommandType.REPEAT_RUNNING, 1);
cmdTypeMap.put(CommandType.RECOVER_SUSPENDED_PROCESS, 1); cmdTypeMap.put(CommandType.RECOVER_SUSPENDED_PROCESS, 1);
cmdTypeMap.put(CommandType.START_FAILURE_TASK_PROCESS, 1); cmdTypeMap.put(CommandType.START_FAILURE_TASK_PROCESS, 1);
@ -296,9 +297,6 @@ public class ProcessService {
/** /**
* get task node list by definitionId * get task node list by definitionId
*
* @param defineId
* @return
*/ */
public List<TaskNode> getTaskNodeListByDefinitionId(Integer defineId) { public List<TaskNode> getTaskNodeListByDefinitionId(Integer defineId) {
ProcessDefinition processDefinition = processDefineMapper.selectById(defineId); ProcessDefinition processDefinition = processDefineMapper.selectById(defineId);
@ -425,7 +423,7 @@ public class ProcessService {
* recursive query sub process definition id by parent id. * recursive query sub process definition id by parent id.
* *
* @param parentId parentId * @param parentId parentId
* @param ids ids * @param ids ids
*/ */
public void recurseFindSubProcessId(int parentId, List<Integer> ids) { public void recurseFindSubProcessId(int parentId, List<Integer> ids) {
ProcessDefinition processDefinition = processDefineMapper.selectById(parentId); ProcessDefinition processDefinition = processDefineMapper.selectById(parentId);
@ -435,7 +433,7 @@ public class ProcessService {
List<TaskNode> taskNodeList = processData.getTasks(); List<TaskNode> taskNodeList = processData.getTasks();
if (taskNodeList != null && taskNodeList.size() > 0) { if (taskNodeList != null && !taskNodeList.isEmpty()) {
for (TaskNode taskNode : taskNodeList) { for (TaskNode taskNode : taskNodeList) {
String parameter = taskNode.getParams(); String parameter = taskNode.getParams();
@ -456,7 +454,7 @@ public class ProcessService {
* create recovery waiting thread command and delete origin command at the same time. * create recovery waiting thread command and delete origin command at the same time.
* if the recovery command is exists, only update the field update_time * if the recovery command is exists, only update the field update_time
* *
* @param originCommand originCommand * @param originCommand originCommand
* @param processInstance processInstance * @param processInstance processInstance
*/ */
public void createRecoveryWaitingThreadCommand(Command originCommand, ProcessInstance processInstance) { public void createRecoveryWaitingThreadCommand(Command originCommand, ProcessInstance processInstance) {
@ -473,17 +471,17 @@ public class ProcessService {
// process instance quit by "waiting thread" state // process instance quit by "waiting thread" state
if (originCommand == null) { if (originCommand == null) {
Command command = new Command( Command command = new Command(
CommandType.RECOVER_WAITTING_THREAD, CommandType.RECOVER_WAITTING_THREAD,
processInstance.getTaskDependType(), processInstance.getTaskDependType(),
processInstance.getFailureStrategy(), processInstance.getFailureStrategy(),
processInstance.getExecutorId(), processInstance.getExecutorId(),
processInstance.getProcessDefinitionId(), processInstance.getProcessDefinitionId(),
JSONUtils.toJsonString(cmdParam), JSONUtils.toJsonString(cmdParam),
processInstance.getWarningType(), processInstance.getWarningType(),
processInstance.getWarningGroupId(), processInstance.getWarningGroupId(),
processInstance.getScheduleTime(), processInstance.getScheduleTime(),
processInstance.getWorkerGroup(), processInstance.getWorkerGroup(),
processInstance.getProcessInstancePriority() processInstance.getProcessInstancePriority()
); );
saveCommand(command); saveCommand(command);
return; return;
@ -508,16 +506,14 @@ public class ProcessService {
/** /**
* get schedule time from command * get schedule time from command
* *
* @param command command * @param command command
* @param cmdParam cmdParam map * @param cmdParam cmdParam map
* @return date * @return date
*/ */
private Date getScheduleTime(Command command, Map<String, String> cmdParam) { private Date getScheduleTime(Command command, Map<String, String> cmdParam) {
Date scheduleTime = command.getScheduleTime(); Date scheduleTime = command.getScheduleTime();
if (scheduleTime == null) { if (scheduleTime == null && cmdParam != null && cmdParam.containsKey(CMDPARAM_COMPLEMENT_DATA_START_DATE)) {
if (cmdParam != null && cmdParam.containsKey(CMDPARAM_COMPLEMENT_DATA_START_DATE)) { scheduleTime = DateUtils.stringToDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE));
scheduleTime = DateUtils.stringToDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE));
}
} }
return scheduleTime; return scheduleTime;
} }
@ -526,8 +522,8 @@ public class ProcessService {
* generate a new work process instance from command. * generate a new work process instance from command.
* *
* @param processDefinition processDefinition * @param processDefinition processDefinition
* @param command command * @param command command
* @param cmdParam cmdParam map * @param cmdParam cmdParam map
* @return process instance * @return process instance
*/ */
private ProcessInstance generateNewProcessInstance(ProcessDefinition processDefinition, private ProcessInstance generateNewProcessInstance(ProcessDefinition processDefinition,
@ -580,10 +576,10 @@ public class ProcessService {
// curing global params // curing global params
processInstance.setGlobalParams(ParameterUtils.curingGlobalParams( processInstance.setGlobalParams(ParameterUtils.curingGlobalParams(
processDefinition.getGlobalParamMap(), processDefinition.getGlobalParamMap(),
processDefinition.getGlobalParamList(), processDefinition.getGlobalParamList(),
getCommandTypeIfComplement(processInstance, command), getCommandTypeIfComplement(processInstance, command),
processInstance.getScheduleTime())); processInstance.getScheduleTime()));
//copy process define json to process instance //copy process define json to process instance
processInstance.setProcessInstanceJson(processDefinition.getProcessDefinitionJson()); processInstance.setProcessInstanceJson(processDefinition.getProcessDefinitionJson());
@ -603,7 +599,7 @@ public class ProcessService {
* use definition creator's tenant. * use definition creator's tenant.
* *
* @param tenantId tenantId * @param tenantId tenantId
* @param userId userId * @param userId userId
* @return tenant * @return tenant
*/ */
public Tenant getTenantForProcess(int tenantId, int userId) { public Tenant getTenantForProcess(int tenantId, int userId) {
@ -626,15 +622,15 @@ public class ProcessService {
/** /**
* check command parameters is valid * check command parameters is valid
* *
* @param command command * @param command command
* @param cmdParam cmdParam map * @param cmdParam cmdParam map
* @return whether command param is valid * @return whether command param is valid
*/ */
private Boolean checkCmdParam(Command command, Map<String, String> cmdParam) { private Boolean checkCmdParam(Command command, Map<String, String> cmdParam) {
if (command.getTaskDependType() == TaskDependType.TASK_ONLY || command.getTaskDependType() == TaskDependType.TASK_PRE) { if (command.getTaskDependType() == TaskDependType.TASK_ONLY || command.getTaskDependType() == TaskDependType.TASK_PRE) {
if (cmdParam == null if (cmdParam == null
|| !cmdParam.containsKey(Constants.CMD_PARAM_START_NODE_NAMES) || !cmdParam.containsKey(Constants.CMD_PARAM_START_NODE_NAMES)
|| cmdParam.get(Constants.CMD_PARAM_START_NODE_NAMES).isEmpty()) { || cmdParam.get(Constants.CMD_PARAM_START_NODE_NAMES).isEmpty()) {
logger.error("command node depend type is {}, but start nodes is null ", command.getTaskDependType()); logger.error("command node depend type is {}, but start nodes is null ", command.getTaskDependType());
return false; return false;
} }
@ -646,7 +642,7 @@ public class ProcessService {
* construct process instance according to one command. * construct process instance according to one command.
* *
* @param command command * @param command command
* @param host host * @param host host
* @return process instance * @return process instance
*/ */
private ProcessInstance constructProcessInstance(Command command, String host) { private ProcessInstance constructProcessInstance(Command command, String host) {
@ -714,7 +710,7 @@ public class ProcessService {
// generate one new process instance // generate one new process instance
processInstance = generateNewProcessInstance(processDefinition, command, cmdParam); processInstance = generateNewProcessInstance(processDefinition, command, cmdParam);
} }
if (!checkCmdParam(command, cmdParam)) { if (Boolean.FALSE.equals(checkCmdParam(command, cmdParam))) {
logger.error("command parameter check failed!"); logger.error("command parameter check failed!");
return null; return null;
} }
@ -742,7 +738,7 @@ public class ProcessService {
initTaskInstance(this.findTaskInstanceById(taskId)); initTaskInstance(this.findTaskInstanceById(taskId));
} }
cmdParam.put(Constants.CMD_PARAM_RECOVERY_START_NODE_STRING, cmdParam.put(Constants.CMD_PARAM_RECOVERY_START_NODE_STRING,
String.join(Constants.COMMA, convertIntListToString(failedList))); String.join(Constants.COMMA, convertIntListToString(failedList)));
processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam)); processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam));
processInstance.setRunTimes(runTime + 1); processInstance.setRunTimes(runTime + 1);
break; break;
@ -755,7 +751,7 @@ public class ProcessService {
cmdParam.remove(Constants.CMD_PARAM_RECOVERY_START_NODE_STRING); cmdParam.remove(Constants.CMD_PARAM_RECOVERY_START_NODE_STRING);
List<Integer> suspendedNodeList = this.findTaskIdByInstanceState(processInstance.getId(), ExecutionStatus.PAUSE); List<Integer> suspendedNodeList = this.findTaskIdByInstanceState(processInstance.getId(), ExecutionStatus.PAUSE);
List<Integer> stopNodeList = findTaskIdByInstanceState(processInstance.getId(), List<Integer> stopNodeList = findTaskIdByInstanceState(processInstance.getId(),
ExecutionStatus.KILL); ExecutionStatus.KILL);
suspendedNodeList.addAll(stopNodeList); suspendedNodeList.addAll(stopNodeList);
for (Integer taskId : suspendedNodeList) { for (Integer taskId : suspendedNodeList) {
// initialize the pause state // initialize the pause state
@ -809,7 +805,7 @@ public class ProcessService {
* return complement data if the process start with complement data * return complement data if the process start with complement data
* *
* @param processInstance processInstance * @param processInstance processInstance
* @param command command * @param command command
* @return command type * @return command type
*/ */
private CommandType getCommandTypeIfComplement(ProcessInstance processInstance, Command command) { private CommandType getCommandTypeIfComplement(ProcessInstance processInstance, Command command) {
@ -824,8 +820,8 @@ public class ProcessService {
* initialize complement data parameters * initialize complement data parameters
* *
* @param processDefinition processDefinition * @param processDefinition processDefinition
* @param processInstance processInstance * @param processInstance processInstance
* @param cmdParam cmdParam * @param cmdParam cmdParam
*/ */
private void initComplementDataParam(ProcessDefinition processDefinition, private void initComplementDataParam(ProcessDefinition processDefinition,
ProcessInstance processInstance, ProcessInstance processInstance,
@ -835,14 +831,14 @@ public class ProcessService {
} }
Date startComplementTime = DateUtils.parse(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE), Date startComplementTime = DateUtils.parse(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE),
YYYY_MM_DD_HH_MM_SS); YYYY_MM_DD_HH_MM_SS);
if (Flag.NO == processInstance.getIsSubProcess()) { if (Flag.NO == processInstance.getIsSubProcess()) {
processInstance.setScheduleTime(startComplementTime); processInstance.setScheduleTime(startComplementTime);
} }
processInstance.setGlobalParams(ParameterUtils.curingGlobalParams( processInstance.setGlobalParams(ParameterUtils.curingGlobalParams(
processDefinition.getGlobalParamMap(), processDefinition.getGlobalParamMap(),
processDefinition.getGlobalParamList(), processDefinition.getGlobalParamList(),
CommandType.COMPLEMENT_DATA, processInstance.getScheduleTime())); CommandType.COMPLEMENT_DATA, processInstance.getScheduleTime()));
} }
@ -862,7 +858,7 @@ public class ProcessService {
Map<String, String> paramMap = JSONUtils.toMap(cmdParam); Map<String, String> paramMap = JSONUtils.toMap(cmdParam);
// write sub process id into cmd param. // write sub process id into cmd param.
if (paramMap.containsKey(CMD_PARAM_SUB_PROCESS) if (paramMap.containsKey(CMD_PARAM_SUB_PROCESS)
&& CMD_PARAM_EMPTY_SUB_PROCESS.equals(paramMap.get(CMD_PARAM_SUB_PROCESS))) { && CMD_PARAM_EMPTY_SUB_PROCESS.equals(paramMap.get(CMD_PARAM_SUB_PROCESS))) {
paramMap.remove(CMD_PARAM_SUB_PROCESS); paramMap.remove(CMD_PARAM_SUB_PROCESS);
paramMap.put(CMD_PARAM_SUB_PROCESS, String.valueOf(subProcessInstance.getId())); paramMap.put(CMD_PARAM_SUB_PROCESS, String.valueOf(subProcessInstance.getId()));
subProcessInstance.setCommandParam(JSONUtils.toJsonString(paramMap)); subProcessInstance.setCommandParam(JSONUtils.toJsonString(paramMap));
@ -875,7 +871,7 @@ public class ProcessService {
ProcessInstance parentInstance = findProcessInstanceDetailById(Integer.parseInt(parentInstanceId)); ProcessInstance parentInstance = findProcessInstanceDetailById(Integer.parseInt(parentInstanceId));
if (parentInstance != null) { if (parentInstance != null) {
subProcessInstance.setGlobalParams( subProcessInstance.setGlobalParams(
joinGlobalParams(parentInstance.getGlobalParams(), subProcessInstance.getGlobalParams())); joinGlobalParams(parentInstance.getGlobalParams(), subProcessInstance.getGlobalParams()));
this.saveProcessInstance(subProcessInstance); this.saveProcessInstance(subProcessInstance);
} else { } else {
logger.error("sub process command params error, cannot find parent instance: {} ", cmdParam); logger.error("sub process command params error, cannot find parent instance: {} ", cmdParam);
@ -897,7 +893,7 @@ public class ProcessService {
* only the keys doesn't in sub process global would be joined. * only the keys doesn't in sub process global would be joined.
* *
* @param parentGlobalParams parentGlobalParams * @param parentGlobalParams parentGlobalParams
* @param subGlobalParams subGlobalParams * @param subGlobalParams subGlobalParams
* @return global params join * @return global params join
*/ */
private String joinGlobalParams(String parentGlobalParams, String subGlobalParams) { private String joinGlobalParams(String parentGlobalParams, String subGlobalParams) {
@ -922,12 +918,11 @@ public class ProcessService {
*/ */
private void initTaskInstance(TaskInstance taskInstance) { private void initTaskInstance(TaskInstance taskInstance) {
if (!taskInstance.isSubProcess()) { if (!taskInstance.isSubProcess()
if (taskInstance.getState().typeIsCancel() || taskInstance.getState().typeIsFailure()) { && (taskInstance.getState().typeIsCancel() || taskInstance.getState().typeIsFailure())) {
taskInstance.setFlag(Flag.NO); taskInstance.setFlag(Flag.NO);
updateTaskInstance(taskInstance); updateTaskInstance(taskInstance);
return; return;
}
} }
taskInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS); taskInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS);
updateTaskInstance(taskInstance); updateTaskInstance(taskInstance);
@ -944,12 +939,12 @@ public class ProcessService {
public TaskInstance submitTask(TaskInstance taskInstance) { public TaskInstance submitTask(TaskInstance taskInstance) {
ProcessInstance processInstance = this.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); ProcessInstance processInstance = this.findProcessInstanceDetailById(taskInstance.getProcessInstanceId());
logger.info("start submit task : {}, instance id:{}, state: {}", logger.info("start submit task : {}, instance id:{}, state: {}",
taskInstance.getName(), taskInstance.getProcessInstanceId(), processInstance.getState()); taskInstance.getName(), taskInstance.getProcessInstanceId(), processInstance.getState());
//submit to db //submit to db
TaskInstance task = submitTaskInstanceToDB(taskInstance, processInstance); TaskInstance task = submitTaskInstanceToDB(taskInstance, processInstance);
if (task == null) { if (task == null) {
logger.error("end submit task to db error, task name:{}, process id:{} state: {} ", logger.error("end submit task to db error, task name:{}, process id:{} state: {} ",
taskInstance.getName(), taskInstance.getProcessInstance(), processInstance.getState()); taskInstance.getName(), taskInstance.getProcessInstance(), processInstance.getState());
return task; return task;
} }
if (!task.getState().typeIsFinished()) { if (!task.getState().typeIsFinished()) {
@ -957,7 +952,7 @@ public class ProcessService {
} }
logger.info("end submit task to db successfully:{} state:{} complete, instance id:{} state: {} ", logger.info("end submit task to db successfully:{} state:{} complete, instance id:{} state: {} ",
taskInstance.getName(), task.getState(), processInstance.getId(), processInstance.getState()); taskInstance.getName(), task.getState(), processInstance.getId(), processInstance.getState());
return task; return task;
} }
@ -968,7 +963,7 @@ public class ProcessService {
* set map {parent instance id, task instance id, 0(child instance id)} * set map {parent instance id, task instance id, 0(child instance id)}
* *
* @param parentInstance parentInstance * @param parentInstance parentInstance
* @param parentTask parentTask * @param parentTask parentTask
* @return process instance map * @return process instance map
*/ */
private ProcessInstanceMap setProcessInstanceMap(ProcessInstance parentInstance, TaskInstance parentTask) { private ProcessInstanceMap setProcessInstanceMap(ProcessInstance parentInstance, TaskInstance parentTask) {
@ -997,7 +992,7 @@ public class ProcessService {
* find previous task work process map. * find previous task work process map.
* *
* @param parentProcessInstance parentProcessInstance * @param parentProcessInstance parentProcessInstance
* @param parentTask parentTask * @param parentTask parentTask
* @return process instance map * @return process instance map
*/ */
private ProcessInstanceMap findPreviousTaskProcessMap(ProcessInstance parentProcessInstance, private ProcessInstanceMap findPreviousTaskProcessMap(ProcessInstance parentProcessInstance,
@ -1015,7 +1010,7 @@ public class ProcessService {
} }
} }
logger.info("sub process instance is not found,parent task:{},parent instance:{}", logger.info("sub process instance is not found,parent task:{},parent instance:{}",
parentTask.getId(), parentProcessInstance.getId()); parentTask.getId(), parentProcessInstance.getId());
return null; return null;
} }
@ -1049,10 +1044,6 @@ public class ProcessService {
/** /**
* complement data needs transform parent parameter to child. * complement data needs transform parent parameter to child.
*
* @param instanceMap
* @param parentProcessInstance
* @return
*/ */
private String getSubWorkFlowParam(ProcessInstanceMap instanceMap, ProcessInstance parentProcessInstance) { private String getSubWorkFlowParam(ProcessInstanceMap instanceMap, ProcessInstance parentProcessInstance) {
// set sub work process command // set sub work process command
@ -1071,11 +1062,6 @@ public class ProcessService {
/** /**
* create sub work process command * create sub work process command
*
* @param parentProcessInstance
* @param childInstance
* @param instanceMap
* @param task
*/ */
public Command createSubProcessCommand(ProcessInstance parentProcessInstance, public Command createSubProcessCommand(ProcessInstance parentProcessInstance,
ProcessInstance childInstance, ProcessInstance childInstance,
@ -1088,25 +1074,23 @@ public class ProcessService {
String processParam = getSubWorkFlowParam(instanceMap, parentProcessInstance); String processParam = getSubWorkFlowParam(instanceMap, parentProcessInstance);
return new Command( return new Command(
commandType, commandType,
TaskDependType.TASK_POST, TaskDependType.TASK_POST,
parentProcessInstance.getFailureStrategy(), parentProcessInstance.getFailureStrategy(),
parentProcessInstance.getExecutorId(), parentProcessInstance.getExecutorId(),
childDefineId, childDefineId,
processParam, processParam,
parentProcessInstance.getWarningType(), parentProcessInstance.getWarningType(),
parentProcessInstance.getWarningGroupId(), parentProcessInstance.getWarningGroupId(),
parentProcessInstance.getScheduleTime(), parentProcessInstance.getScheduleTime(),
task.getWorkerGroup(), task.getWorkerGroup(),
parentProcessInstance.getProcessInstancePriority() parentProcessInstance.getProcessInstancePriority()
); );
} }
/** /**
* initialize sub work flow state * initialize sub work flow state
* child instance state would be initialized when 'recovery from pause/stop/failure' * child instance state would be initialized when 'recovery from pause/stop/failure'
*
* @param childInstance
*/ */
private void initSubInstanceState(ProcessInstance childInstance) { private void initSubInstanceState(ProcessInstance childInstance) {
if (childInstance != null) { if (childInstance != null) {
@ -1119,9 +1103,6 @@ public class ProcessService {
* get sub work flow command type * get sub work flow command type
* child instance exist: child command = fatherCommand * child instance exist: child command = fatherCommand
* child instance not exists: child command = fatherCommand[0] * child instance not exists: child command = fatherCommand[0]
*
* @param parentProcessInstance
* @return
*/ */
private CommandType getSubCommandType(ProcessInstance parentProcessInstance, ProcessInstance childInstance) { private CommandType getSubCommandType(ProcessInstance parentProcessInstance, ProcessInstance childInstance) {
CommandType commandType = parentProcessInstance.getCommandType(); CommandType commandType = parentProcessInstance.getCommandType();
@ -1136,7 +1117,7 @@ public class ProcessService {
* update sub process definition * update sub process definition
* *
* @param parentProcessInstance parentProcessInstance * @param parentProcessInstance parentProcessInstance
* @param childDefinitionId childDefinitionId * @param childDefinitionId childDefinitionId
*/ */
private void updateSubProcessDefinitionByParent(ProcessInstance parentProcessInstance, int childDefinitionId) { private void updateSubProcessDefinitionByParent(ProcessInstance parentProcessInstance, int childDefinitionId) {
ProcessDefinition fatherDefinition = this.findProcessDefineById(parentProcessInstance.getProcessDefinitionId()); ProcessDefinition fatherDefinition = this.findProcessDefineById(parentProcessInstance.getProcessDefinitionId());
@ -1150,7 +1131,7 @@ public class ProcessService {
/** /**
* submit task to mysql * submit task to mysql
* *
* @param taskInstance taskInstance * @param taskInstance taskInstance
* @param processInstance processInstance * @param processInstance processInstance
* @return task instance * @return task instance
*/ */
@ -1163,7 +1144,7 @@ public class ProcessService {
} else { } else {
if (processInstanceState != ExecutionStatus.READY_STOP if (processInstanceState != ExecutionStatus.READY_STOP
&& processInstanceState != ExecutionStatus.READY_PAUSE) { && processInstanceState != ExecutionStatus.READY_PAUSE) {
// failure task set invalid // failure task set invalid
taskInstance.setFlag(Flag.NO); taskInstance.setFlag(Flag.NO);
updateTaskInstance(taskInstance); updateTaskInstance(taskInstance);
@ -1204,19 +1185,19 @@ public class ProcessService {
* return stop if work process state is ready stop * return stop if work process state is ready stop
* if all of above are not satisfied, return submit success * if all of above are not satisfied, return submit success
* *
* @param taskInstance taskInstance * @param taskInstance taskInstance
* @param processInstanceState processInstanceState * @param processInstanceState processInstanceState
* @return process instance state * @return process instance state
*/ */
public ExecutionStatus getSubmitTaskState(TaskInstance taskInstance, ExecutionStatus processInstanceState) { public ExecutionStatus getSubmitTaskState(TaskInstance taskInstance, ExecutionStatus processInstanceState) {
ExecutionStatus state = taskInstance.getState(); ExecutionStatus state = taskInstance.getState();
if ( if (
// running, delayed or killed // running, delayed or killed
// the task already exists in task queue // the task already exists in task queue
// return state // return state
state == ExecutionStatus.RUNNING_EXECUTION state == ExecutionStatus.RUNNING_EXECUTION
|| state == ExecutionStatus.DELAY_EXECUTION || state == ExecutionStatus.DELAY_EXECUTION
|| state == ExecutionStatus.KILL || state == ExecutionStatus.KILL
) { ) {
return state; return state;
} }
@ -1225,7 +1206,7 @@ public class ProcessService {
if (processInstanceState == ExecutionStatus.READY_PAUSE) { if (processInstanceState == ExecutionStatus.READY_PAUSE) {
state = ExecutionStatus.PAUSE; state = ExecutionStatus.PAUSE;
} else if (processInstanceState == ExecutionStatus.READY_STOP } else if (processInstanceState == ExecutionStatus.READY_STOP
|| !checkProcessStrategy(taskInstance)) { || !checkProcessStrategy(taskInstance)) {
state = ExecutionStatus.KILL; state = ExecutionStatus.KILL;
} else { } else {
state = ExecutionStatus.SUBMITTED_SUCCESS; state = ExecutionStatus.SUBMITTED_SUCCESS;
@ -1380,7 +1361,7 @@ public class ProcessService {
* get id list by task state * get id list by task state
* *
* @param instanceId instanceId * @param instanceId instanceId
* @param state state * @param state state
* @return task instance states * @return task instance states
*/ */
public List<Integer> findTaskIdByInstanceState(int instanceId, ExecutionStatus state) { public List<Integer> findTaskIdByInstanceState(int instanceId, ExecutionStatus state) {
@ -1435,7 +1416,7 @@ public class ProcessService {
* find work process map by parent process id and parent task id. * find work process map by parent process id and parent task id.
* *
* @param parentWorkProcessId parentWorkProcessId * @param parentWorkProcessId parentWorkProcessId
* @param parentTaskId parentTaskId * @param parentTaskId parentTaskId
* @return process instance map * @return process instance map
*/ */
public ProcessInstanceMap findWorkProcessMapByParent(Integer parentWorkProcessId, Integer parentTaskId) { public ProcessInstanceMap findWorkProcessMapByParent(Integer parentWorkProcessId, Integer parentTaskId) {
@ -1457,7 +1438,7 @@ public class ProcessService {
* find sub process instance * find sub process instance
* *
* @param parentProcessId parentProcessId * @param parentProcessId parentProcessId
* @param parentTaskId parentTaskId * @param parentTaskId parentTaskId
* @return process instance * @return process instance
*/ */
public ProcessInstance findSubProcessInstance(Integer parentProcessId, Integer parentTaskId) { public ProcessInstance findSubProcessInstance(Integer parentProcessId, Integer parentTaskId) {
@ -1489,12 +1470,12 @@ public class ProcessService {
/** /**
* change task state * change task state
* *
* @param state state * @param state state
* @param startTime startTime * @param startTime startTime
* @param host host * @param host host
* @param executePath executePath * @param executePath executePath
* @param logPath logPath * @param logPath logPath
* @param taskInstId taskInstId * @param taskInstId taskInstId
*/ */
public void changeTaskState(TaskInstance taskInstance, ExecutionStatus state, Date startTime, String host, public void changeTaskState(TaskInstance taskInstance, ExecutionStatus state, Date startTime, String host,
String executePath, String executePath,
@ -1522,12 +1503,12 @@ public class ProcessService {
* update the process instance * update the process instance
* *
* @param processInstanceId processInstanceId * @param processInstanceId processInstanceId
* @param processJson processJson * @param processJson processJson
* @param globalParams globalParams * @param globalParams globalParams
* @param scheduleTime scheduleTime * @param scheduleTime scheduleTime
* @param flag flag * @param flag flag
* @param locations locations * @param locations locations
* @param connects connects * @param connects connects
* @return update process instance result * @return update process instance result
*/ */
public int updateProcessInstance(Integer processInstanceId, String processJson, public int updateProcessInstance(Integer processInstanceId, String processJson,
@ -1548,10 +1529,10 @@ public class ProcessService {
/** /**
* change task state * change task state
* *
* @param state state * @param state state
* @param endTime endTime * @param endTime endTime
* @param taskInstId taskInstId * @param taskInstId taskInstId
* @param varPool varPool * @param varPool varPool
*/ */
public void changeTaskState(TaskInstance taskInstance, ExecutionStatus state, public void changeTaskState(TaskInstance taskInstance, ExecutionStatus state,
Date endTime, Date endTime,
@ -1577,7 +1558,7 @@ public class ProcessService {
if (intList == null) { if (intList == null) {
return new ArrayList<>(); return new ArrayList<>();
} }
List<String> result = new ArrayList<String>(intList.size()); List<String> result = new ArrayList<>(intList.size());
for (Integer intVar : intList) { for (Integer intVar : intList) {
result.add(String.valueOf(intVar)); result.add(String.valueOf(intVar));
} }
@ -1642,7 +1623,7 @@ public class ProcessService {
*/ */
public List<TaskInstance> queryNeedFailoverTaskInstances(String host) { public List<TaskInstance> queryNeedFailoverTaskInstances(String host) {
return taskInstanceMapper.queryByHostAndStatus(host, return taskInstanceMapper.queryByHostAndStatus(host,
stateArray); stateArray);
} }
/** /**
@ -1659,7 +1640,7 @@ public class ProcessService {
* update process instance state by id * update process instance state by id
* *
* @param processInstanceId processInstanceId * @param processInstanceId processInstanceId
* @param executionStatus executionStatus * @param executionStatus executionStatus
* @return update process result * @return update process result
*/ */
public int updateProcessInstanceState(Integer processInstanceId, ExecutionStatus executionStatus) { public int updateProcessInstanceState(Integer processInstanceId, ExecutionStatus executionStatus) {
@ -1696,7 +1677,7 @@ public class ProcessService {
/** /**
* find tenant code by resource name * find tenant code by resource name
* *
* @param resName resource name * @param resName resource name
* @param resourceType resource type * @param resourceType resource type
* @return tenant code * @return tenant code
*/ */
@ -1714,35 +1695,35 @@ public class ProcessService {
*/ */
public List<Schedule> selectAllByProcessDefineId(int[] ids) { public List<Schedule> selectAllByProcessDefineId(int[] ids) {
return scheduleMapper.selectAllByProcessDefineArray( return scheduleMapper.selectAllByProcessDefineArray(
ids); ids);
} }
/** /**
* get dependency cycle by work process define id and scheduler fire time * get dependency cycle by work process define id and scheduler fire time
* *
* @param masterId masterId * @param masterId masterId
* @param processDefinitionId processDefinitionId * @param processDefinitionId processDefinitionId
* @param scheduledFireTime the time the task schedule is expected to trigger * @param scheduledFireTime the time the task schedule is expected to trigger
* @return CycleDependency * @return CycleDependency
* @throws Exception if error throws Exception * @throws Exception if error throws Exception
*/ */
public CycleDependency getCycleDependency(int masterId, int processDefinitionId, Date scheduledFireTime) throws Exception { public CycleDependency getCycleDependency(int masterId, int processDefinitionId, Date scheduledFireTime) throws Exception {
List<CycleDependency> list = getCycleDependencies(masterId, new int[] {processDefinitionId}, scheduledFireTime); List<CycleDependency> list = getCycleDependencies(masterId, new int[]{processDefinitionId}, scheduledFireTime);
return list.size() > 0 ? list.get(0) : null; return !list.isEmpty() ? list.get(0) : null;
} }
/** /**
* get dependency cycle list by work process define id list and scheduler fire time * get dependency cycle list by work process define id list and scheduler fire time
* *
* @param masterId masterId * @param masterId masterId
* @param ids ids * @param ids ids
* @param scheduledFireTime the time the task schedule is expected to trigger * @param scheduledFireTime the time the task schedule is expected to trigger
* @return CycleDependency list * @return CycleDependency list
* @throws Exception if error throws Exception * @throws Exception if error throws Exception
*/ */
public List<CycleDependency> getCycleDependencies(int masterId, int[] ids, Date scheduledFireTime) throws Exception { public List<CycleDependency> getCycleDependencies(int masterId, int[] ids, Date scheduledFireTime) throws Exception {
List<CycleDependency> cycleDependencyList = new ArrayList<CycleDependency>(); List<CycleDependency> cycleDependencyList = new ArrayList<>();
if (null == ids || ids.length == 0) { if (null == ids || ids.length == 0) {
logger.warn("ids[] is empty!is invalid!"); logger.warn("ids[] is empty!is invalid!");
return cycleDependencyList; return cycleDependencyList;
@ -1769,14 +1750,10 @@ public class ProcessService {
} }
Calendar calendar = Calendar.getInstance(); Calendar calendar = Calendar.getInstance();
switch (cycleEnum) { switch (cycleEnum) {
/*case MINUTE:
calendar.add(Calendar.MINUTE,-61);*/
case HOUR: case HOUR:
calendar.add(Calendar.HOUR, -25); calendar.add(Calendar.HOUR, -25);
break; break;
case DAY: case DAY:
calendar.add(Calendar.DATE, -32);
break;
case WEEK: case WEEK:
calendar.add(Calendar.DATE, -32); calendar.add(Calendar.DATE, -32);
break; break;
@ -1784,7 +1761,8 @@ public class ProcessService {
calendar.add(Calendar.MONTH, -13); calendar.add(Calendar.MONTH, -13);
break; break;
default: default:
logger.warn("Dependent process definition's cycleEnum is {},not support!!", cycleEnum.name()); String cycleName = cycleEnum.name();
logger.warn("Dependent process definition's cycleEnum is {},not support!!", cycleName);
continue; continue;
} }
Date start = calendar.getTime(); Date start = calendar.getTime();
@ -1794,7 +1772,7 @@ public class ProcessService {
} else { } else {
list = CronUtils.getFireDateList(start, scheduledFireTime, depCronExpression); list = CronUtils.getFireDateList(start, scheduledFireTime, depCronExpression);
} }
if (list.size() >= 1) { if (!list.isEmpty()) {
start = list.get(list.size() - 1); start = list.get(list.size() - 1);
CycleDependency dependency = new CycleDependency(depSchedule.getProcessDefinitionId(), start, CronUtils.getExpirationTime(start, cycleEnum), cycleEnum); CycleDependency dependency = new CycleDependency(depSchedule.getProcessDefinitionId(), start, CronUtils.getExpirationTime(start, cycleEnum), cycleEnum);
cycleDependencyList.add(dependency); cycleDependencyList.add(dependency);
@ -1813,8 +1791,8 @@ public class ProcessService {
*/ */
public ProcessInstance findLastSchedulerProcessInterval(int definitionId, DateInterval dateInterval) { public ProcessInstance findLastSchedulerProcessInterval(int definitionId, DateInterval dateInterval) {
return processInstanceMapper.queryLastSchedulerProcess(definitionId, return processInstanceMapper.queryLastSchedulerProcess(definitionId,
dateInterval.getStartTime(), dateInterval.getStartTime(),
dateInterval.getEndTime()); dateInterval.getEndTime());
} }
/** /**
@ -1826,23 +1804,23 @@ public class ProcessService {
*/ */
public ProcessInstance findLastManualProcessInterval(int definitionId, DateInterval dateInterval) { public ProcessInstance findLastManualProcessInterval(int definitionId, DateInterval dateInterval) {
return processInstanceMapper.queryLastManualProcess(definitionId, return processInstanceMapper.queryLastManualProcess(definitionId,
dateInterval.getStartTime(), dateInterval.getStartTime(),
dateInterval.getEndTime()); dateInterval.getEndTime());
} }
/** /**
* find last running process instance * find last running process instance
* *
* @param definitionId process definition id * @param definitionId process definition id
* @param startTime start time * @param startTime start time
* @param endTime end time * @param endTime end time
* @return process instance * @return process instance
*/ */
public ProcessInstance findLastRunningProcess(int definitionId, Date startTime, Date endTime) { public ProcessInstance findLastRunningProcess(int definitionId, Date startTime, Date endTime) {
return processInstanceMapper.queryLastRunningProcess(definitionId, return processInstanceMapper.queryLastRunningProcess(definitionId,
startTime, startTime,
endTime, endTime,
stateArray); stateArray);
} }
/** /**
@ -1867,6 +1845,7 @@ public class ProcessService {
/** /**
* query project name and user name by processInstanceId. * query project name and user name by processInstanceId.
*
* @param processInstanceId processInstanceId * @param processInstanceId processInstanceId
* @return projectName and userName * @return projectName and userName
*/ */
@ -1934,35 +1913,32 @@ public class ProcessService {
/** /**
* list unauthorized udf function * list unauthorized udf function
* *
* @param userId user id * @param userId user id
* @param needChecks data source id array * @param needChecks data source id array
* @return unauthorized udf function list * @return unauthorized udf function list
*/ */
public <T> List<T> listUnauthorized(int userId, T[] needChecks, AuthorizationType authorizationType) { public <T> List<T> listUnauthorized(int userId, T[] needChecks, AuthorizationType authorizationType) {
List<T> resultList = new ArrayList<T>(); List<T> resultList = new ArrayList<>();
if (Objects.nonNull(needChecks) && needChecks.length > 0) { if (Objects.nonNull(needChecks) && needChecks.length > 0) {
Set<T> originResSet = new HashSet<T>(Arrays.asList(needChecks)); Set<T> originResSet = new HashSet<>(Arrays.asList(needChecks));
switch (authorizationType) { switch (authorizationType) {
case RESOURCE_FILE_ID: case RESOURCE_FILE_ID:
Set<Integer> authorizedResourceFiles = resourceMapper.listAuthorizedResourceById(userId, needChecks).stream().map(t -> t.getId()).collect(toSet()); case UDF_FILE:
Set<Integer> authorizedResourceFiles = resourceMapper.listAuthorizedResourceById(userId, needChecks).stream().map(Resource::getId).collect(toSet());
originResSet.removeAll(authorizedResourceFiles); originResSet.removeAll(authorizedResourceFiles);
break; break;
case RESOURCE_FILE_NAME: case RESOURCE_FILE_NAME:
Set<String> authorizedResources = resourceMapper.listAuthorizedResource(userId, needChecks).stream().map(t -> t.getFullName()).collect(toSet()); Set<String> authorizedResources = resourceMapper.listAuthorizedResource(userId, needChecks).stream().map(Resource::getFullName).collect(toSet());
originResSet.removeAll(authorizedResources); originResSet.removeAll(authorizedResources);
break; break;
case UDF_FILE:
Set<Integer> authorizedUdfFiles = resourceMapper.listAuthorizedResourceById(userId, needChecks).stream().map(t -> t.getId()).collect(toSet());
originResSet.removeAll(authorizedUdfFiles);
break;
case DATASOURCE: case DATASOURCE:
Set<Integer> authorizedDatasources = dataSourceMapper.listAuthorizedDataSource(userId, needChecks).stream().map(t -> t.getId()).collect(toSet()); Set<Integer> authorizedDatasources = dataSourceMapper.listAuthorizedDataSource(userId, needChecks).stream().map(DataSource::getId).collect(toSet());
originResSet.removeAll(authorizedDatasources); originResSet.removeAll(authorizedDatasources);
break; break;
case UDF: case UDF:
Set<Integer> authorizedUdfs = udfFuncMapper.listAuthorizedUdfFunc(userId, needChecks).stream().map(t -> t.getId()).collect(toSet()); Set<Integer> authorizedUdfs = udfFuncMapper.listAuthorizedUdfFunc(userId, needChecks).stream().map(UdfFunc::getId).collect(toSet());
originResSet.removeAll(authorizedUdfs); originResSet.removeAll(authorizedUdfs);
break; break;
default: default:
@ -2007,9 +1983,6 @@ public class ProcessService {
/** /**
* format task app id in task instance * format task app id in task instance
*
* @param taskInstance
* @return
*/ */
public String formatTaskAppId(TaskInstance taskInstance) { public String formatTaskAppId(TaskInstance taskInstance) {
ProcessDefinition definition = this.findProcessDefineById(taskInstance.getProcessDefinitionId()); ProcessDefinition definition = this.findProcessDefineById(taskInstance.getProcessDefinitionId());
@ -2019,9 +1992,9 @@ public class ProcessService {
return ""; return "";
} }
return String.format("%s_%s_%s", return String.format("%s_%s_%s",
definition.getId(), definition.getId(),
processInstanceById.getId(), processInstanceById.getId(),
taskInstance.getId()); taskInstance.getId());
} }
} }

15
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/ProcessScheduleJob.java
Unescape View File

@ -14,8 +14,8 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.service.quartz;
package org.apache.dolphinscheduler.service.quartz;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.CommandType;
@ -25,6 +25,9 @@ import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import java.util.Date;
import org.quartz.Job; import org.quartz.Job;
import org.quartz.JobDataMap; import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext; import org.quartz.JobExecutionContext;
@ -34,8 +37,6 @@ import org.slf4j.LoggerFactory;
import org.springframework.util.Assert; import org.springframework.util.Assert;
import org.springframework.util.StringUtils; import org.springframework.util.StringUtils;
import java.util.Date;
/** /**
* process schedule job * process schedule job
*/ */
@ -46,7 +47,7 @@ public class ProcessScheduleJob implements Job {
*/ */
private static final Logger logger = LoggerFactory.getLogger(ProcessScheduleJob.class); private static final Logger logger = LoggerFactory.getLogger(ProcessScheduleJob.class);
public ProcessService getProcessService(){ public ProcessService getProcessService() {
return SpringApplicationContext.getBean(ProcessService.class); return SpringApplicationContext.getBean(ProcessService.class);
} }
@ -66,10 +67,8 @@ public class ProcessScheduleJob implements Job {
int projectId = dataMap.getInt(Constants.PROJECT_ID); int projectId = dataMap.getInt(Constants.PROJECT_ID);
int scheduleId = dataMap.getInt(Constants.SCHEDULE_ID); int scheduleId = dataMap.getInt(Constants.SCHEDULE_ID);
Date scheduledFireTime = context.getScheduledFireTime(); Date scheduledFireTime = context.getScheduledFireTime();
Date fireTime = context.getFireTime(); Date fireTime = context.getFireTime();
logger.info("scheduled fire time :{}, fire time :{}, process id :{}", scheduledFireTime, fireTime, scheduleId); logger.info("scheduled fire time :{}, fire time :{}, process id :{}", scheduledFireTime, fireTime, scheduleId);
@ -82,11 +81,10 @@ public class ProcessScheduleJob implements Job {
return; return;
} }
ProcessDefinition processDefinition = getProcessService().findProcessDefineById(schedule.getProcessDefinitionId()); ProcessDefinition processDefinition = getProcessService().findProcessDefineById(schedule.getProcessDefinitionId());
// release state : online/offline // release state : online/offline
ReleaseState releaseState = processDefinition.getReleaseState(); ReleaseState releaseState = processDefinition.getReleaseState();
if (processDefinition == null || releaseState == ReleaseState.OFFLINE) { if (releaseState == ReleaseState.OFFLINE) {
logger.warn("process definition does not exist in db or offline，need not to create command, projectId:{}, processId:{}", projectId, scheduleId); logger.warn("process definition does not exist in db or offline，need not to create command, projectId:{}, processId:{}", projectId, scheduleId);
return; return;
} }
@ -107,7 +105,6 @@ public class ProcessScheduleJob implements Job {
getProcessService().createCommand(command); getProcessService().createCommand(command);
} }
/** /**
* delete job * delete job
*/ */

614
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java
Unescape View File

@ -14,15 +14,76 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.service.quartz; package org.apache.dolphinscheduler.service.quartz;
import static org.apache.dolphinscheduler.common.Constants.ORG_POSTGRESQL_DRIVER;
import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_DATASOURCE_MYDS_CONNECTIONPROVIDER_CLASS;
import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_JOBSTORE_ACQUIRETRIGGERSWITHINLOCK;
import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_JOBSTORE_CLASS;
import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_JOBSTORE_CLUSTERCHECKININTERVAL;
import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_JOBSTORE_DATASOURCE;
import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_JOBSTORE_DRIVERDELEGATECLASS;
import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_JOBSTORE_ISCLUSTERED;
import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_JOBSTORE_MISFIRETHRESHOLD;
import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_JOBSTORE_TABLEPREFIX;
import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_JOBSTORE_USEPROPERTIES;
import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_SCHEDULER_INSTANCEID;
import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_SCHEDULER_INSTANCENAME;
import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_SCHEDULER_MAKESCHEDULERTHREADDAEMON;
import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_THREADPOOL_CLASS;
import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_THREADPOOL_MAKETHREADSDAEMONS;
import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_THREADPOOL_THREADCOUNT;
import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_THREADPOOL_THREADPRIORITY;
import static org.apache.dolphinscheduler.common.Constants.PROJECT_ID;
import static org.apache.dolphinscheduler.common.Constants.QUARTZ_ACQUIRETRIGGERSWITHINLOCK;
import static org.apache.dolphinscheduler.common.Constants.QUARTZ_CLUSTERCHECKININTERVAL;
import static org.apache.dolphinscheduler.common.Constants.QUARTZ_DATASOURCE;
import static org.apache.dolphinscheduler.common.Constants.QUARTZ_INSTANCEID;
import static org.apache.dolphinscheduler.common.Constants.QUARTZ_INSTANCENAME;
import static org.apache.dolphinscheduler.common.Constants.QUARTZ_JOB_GROUP_PRIFIX;
import static org.apache.dolphinscheduler.common.Constants.QUARTZ_JOB_PRIFIX;
import static org.apache.dolphinscheduler.common.Constants.QUARTZ_MISFIRETHRESHOLD;
import static org.apache.dolphinscheduler.common.Constants.QUARTZ_PROPERTIES_PATH;
import static org.apache.dolphinscheduler.common.Constants.QUARTZ_TABLE_PREFIX;
import static org.apache.dolphinscheduler.common.Constants.QUARTZ_THREADCOUNT;
import static org.apache.dolphinscheduler.common.Constants.QUARTZ_THREADPRIORITY;
import static org.apache.dolphinscheduler.common.Constants.SCHEDULE;
import static org.apache.dolphinscheduler.common.Constants.SCHEDULE_ID;
import static org.apache.dolphinscheduler.common.Constants.SPRING_DATASOURCE_DRIVER_CLASS_NAME;
import static org.apache.dolphinscheduler.common.Constants.STRING_FALSE;
import static org.apache.dolphinscheduler.common.Constants.STRING_TRUE;
import static org.apache.dolphinscheduler.common.Constants.UNDERLINE;
import static org.quartz.CronScheduleBuilder.cronSchedule;
import static org.quartz.JobBuilder.newJob;
import static org.quartz.TriggerBuilder.newTrigger;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.service.exceptions.ServiceException;
import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.utils.*; import java.util.ArrayList;
import org.apache.dolphinscheduler.dao.entity.Schedule; import java.util.Date;
import org.quartz.*; import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.quartz.CronTrigger;
import org.quartz.Job;
import org.quartz.JobDetail;
import org.quartz.JobKey;
import org.quartz.Scheduler;
import org.quartz.SchedulerException;
import org.quartz.TriggerKey;
import org.quartz.impl.StdSchedulerFactory; import org.quartz.impl.StdSchedulerFactory;
import org.quartz.impl.jdbcjobstore.JobStoreTX; import org.quartz.impl.jdbcjobstore.JobStoreTX;
import org.quartz.impl.jdbcjobstore.PostgreSQLDelegate; import org.quartz.impl.jdbcjobstore.PostgreSQLDelegate;
@ -32,300 +93,289 @@ import org.quartz.simpl.SimpleThreadPool;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import static org.apache.dolphinscheduler.common.Constants.*;
import static org.quartz.CronScheduleBuilder.cronSchedule;
import static org.quartz.JobBuilder.newJob;
import static org.quartz.TriggerBuilder.newTrigger;
/** /**
* single Quartz executors instance * single Quartz executors instance
*/ */
public class QuartzExecutors { public class QuartzExecutors {
/** /**
* logger of QuartzExecutors * logger of QuartzExecutors
*/ */
private static final Logger logger = LoggerFactory.getLogger(QuartzExecutors.class); private static final Logger logger = LoggerFactory.getLogger(QuartzExecutors.class);
/** /**
* read write lock * read write lock
*/ */
private final ReadWriteLock lock = new ReentrantReadWriteLock(); private final ReadWriteLock lock = new ReentrantReadWriteLock();
/** /**
* A Scheduler maintains a registry of org.quartz.JobDetail and Trigger. * A Scheduler maintains a registry of org.quartz.JobDetail and Trigger.
*/ */
private static Scheduler scheduler; private static Scheduler scheduler;
/** /**
* load conf * load conf
*/ */
private static Configuration conf; private static Configuration conf;
private static final class Holder { private static final class Holder {
private static final QuartzExecutors instance = new QuartzExecutors(); private static final QuartzExecutors instance = new QuartzExecutors();
}
private QuartzExecutors() {
try {
conf = new PropertiesConfiguration(QUARTZ_PROPERTIES_PATH);
init();
}catch (ConfigurationException e){
logger.warn("not loaded quartz configuration file, will used default value",e);
} }
}
private QuartzExecutors() {
/** try {
* thread safe and performance promote conf = new PropertiesConfiguration(QUARTZ_PROPERTIES_PATH);
* @return instance of Quartz Executors init();
*/ } catch (ConfigurationException e) {
public static QuartzExecutors getInstance() { logger.warn("not loaded quartz configuration file, will used default value", e);
return Holder.instance; }
}
/**
* init
*
* Returns a client-usable handle to a Scheduler.
*/
private void init() {
try {
StdSchedulerFactory schedulerFactory = new StdSchedulerFactory();
Properties properties = new Properties();
String dataSourceDriverClass = org.apache.dolphinscheduler.dao.utils.PropertyUtils.getString(SPRING_DATASOURCE_DRIVER_CLASS_NAME);
if (dataSourceDriverClass.equals(ORG_POSTGRESQL_DRIVER)){
properties.setProperty(ORG_QUARTZ_JOBSTORE_DRIVERDELEGATECLASS,conf.getString(ORG_QUARTZ_JOBSTORE_DRIVERDELEGATECLASS, PostgreSQLDelegate.class.getName()));
} else {
properties.setProperty(ORG_QUARTZ_JOBSTORE_DRIVERDELEGATECLASS,conf.getString(ORG_QUARTZ_JOBSTORE_DRIVERDELEGATECLASS, StdJDBCDelegate.class.getName()));
}
properties.setProperty(ORG_QUARTZ_SCHEDULER_INSTANCENAME, conf.getString(ORG_QUARTZ_SCHEDULER_INSTANCENAME, QUARTZ_INSTANCENAME));
properties.setProperty(ORG_QUARTZ_SCHEDULER_INSTANCEID, conf.getString(ORG_QUARTZ_SCHEDULER_INSTANCEID, QUARTZ_INSTANCEID));
properties.setProperty(ORG_QUARTZ_SCHEDULER_MAKESCHEDULERTHREADDAEMON,conf.getString(ORG_QUARTZ_SCHEDULER_MAKESCHEDULERTHREADDAEMON,STRING_TRUE));
properties.setProperty(ORG_QUARTZ_JOBSTORE_USEPROPERTIES,conf.getString(ORG_QUARTZ_JOBSTORE_USEPROPERTIES,STRING_FALSE));
properties.setProperty(ORG_QUARTZ_THREADPOOL_CLASS,conf.getString(ORG_QUARTZ_THREADPOOL_CLASS, SimpleThreadPool.class.getName()));
properties.setProperty(ORG_QUARTZ_THREADPOOL_MAKETHREADSDAEMONS,conf.getString(ORG_QUARTZ_THREADPOOL_MAKETHREADSDAEMONS,STRING_TRUE));
properties.setProperty(ORG_QUARTZ_THREADPOOL_THREADCOUNT,conf.getString(ORG_QUARTZ_THREADPOOL_THREADCOUNT, QUARTZ_THREADCOUNT));
properties.setProperty(ORG_QUARTZ_THREADPOOL_THREADPRIORITY,conf.getString(ORG_QUARTZ_THREADPOOL_THREADPRIORITY, QUARTZ_THREADPRIORITY));
properties.setProperty(ORG_QUARTZ_JOBSTORE_CLASS,conf.getString(ORG_QUARTZ_JOBSTORE_CLASS, JobStoreTX.class.getName()));
properties.setProperty(ORG_QUARTZ_JOBSTORE_TABLEPREFIX,conf.getString(ORG_QUARTZ_JOBSTORE_TABLEPREFIX, QUARTZ_TABLE_PREFIX));
properties.setProperty(ORG_QUARTZ_JOBSTORE_ISCLUSTERED,conf.getString(ORG_QUARTZ_JOBSTORE_ISCLUSTERED,STRING_TRUE));
properties.setProperty(ORG_QUARTZ_JOBSTORE_MISFIRETHRESHOLD,conf.getString(ORG_QUARTZ_JOBSTORE_MISFIRETHRESHOLD, QUARTZ_MISFIRETHRESHOLD));
properties.setProperty(ORG_QUARTZ_JOBSTORE_CLUSTERCHECKININTERVAL,conf.getString(ORG_QUARTZ_JOBSTORE_CLUSTERCHECKININTERVAL, QUARTZ_CLUSTERCHECKININTERVAL));
properties.setProperty(ORG_QUARTZ_JOBSTORE_ACQUIRETRIGGERSWITHINLOCK,conf.getString(ORG_QUARTZ_JOBSTORE_ACQUIRETRIGGERSWITHINLOCK, QUARTZ_ACQUIRETRIGGERSWITHINLOCK));
properties.setProperty(ORG_QUARTZ_JOBSTORE_DATASOURCE,conf.getString(ORG_QUARTZ_JOBSTORE_DATASOURCE, QUARTZ_DATASOURCE));
properties.setProperty(ORG_QUARTZ_DATASOURCE_MYDS_CONNECTIONPROVIDER_CLASS,conf.getString(ORG_QUARTZ_DATASOURCE_MYDS_CONNECTIONPROVIDER_CLASS,DruidConnectionProvider.class.getName()));
schedulerFactory.initialize(properties);
scheduler = schedulerFactory.getScheduler();
} catch (SchedulerException e) {
logger.error(e.getMessage(),e);
System.exit(1);
} }
} /**
* thread safe and performance promote
/** *
* Whether the scheduler has been started. * @return instance of Quartz Executors
* */
* @throws SchedulerException scheduler exception public static QuartzExecutors getInstance() {
*/ return Holder.instance;
public void start() throws SchedulerException {
if (!scheduler.isStarted()){
scheduler.start();
logger.info("Quartz service started" );
} }
}
/**
/** * init
* stop all scheduled tasks * <p>
* * Returns a client-usable handle to a Scheduler.
* Halts the Scheduler's firing of Triggers, */
* and cleans up all resources associated with the Scheduler. private void init() {
* try {
* The scheduler cannot be re-started. StdSchedulerFactory schedulerFactory = new StdSchedulerFactory();
* @throws SchedulerException scheduler exception Properties properties = new Properties();
*/
public void shutdown() throws SchedulerException { String dataSourceDriverClass = org.apache.dolphinscheduler.dao.utils.PropertyUtils.getString(SPRING_DATASOURCE_DRIVER_CLASS_NAME);
if (!scheduler.isShutdown()) { if (dataSourceDriverClass.equals(ORG_POSTGRESQL_DRIVER)) {
// don't wait for the task to complete properties.setProperty(ORG_QUARTZ_JOBSTORE_DRIVERDELEGATECLASS, conf.getString(ORG_QUARTZ_JOBSTORE_DRIVERDELEGATECLASS, PostgreSQLDelegate.class.getName()));
scheduler.shutdown(); } else {
logger.info("Quartz service stopped, and halt all tasks"); properties.setProperty(ORG_QUARTZ_JOBSTORE_DRIVERDELEGATECLASS, conf.getString(ORG_QUARTZ_JOBSTORE_DRIVERDELEGATECLASS, StdJDBCDelegate.class.getName()));
}
properties.setProperty(ORG_QUARTZ_SCHEDULER_INSTANCENAME, conf.getString(ORG_QUARTZ_SCHEDULER_INSTANCENAME, QUARTZ_INSTANCENAME));
properties.setProperty(ORG_QUARTZ_SCHEDULER_INSTANCEID, conf.getString(ORG_QUARTZ_SCHEDULER_INSTANCEID, QUARTZ_INSTANCEID));
properties.setProperty(ORG_QUARTZ_SCHEDULER_MAKESCHEDULERTHREADDAEMON, conf.getString(ORG_QUARTZ_SCHEDULER_MAKESCHEDULERTHREADDAEMON, STRING_TRUE));
properties.setProperty(ORG_QUARTZ_JOBSTORE_USEPROPERTIES, conf.getString(ORG_QUARTZ_JOBSTORE_USEPROPERTIES, STRING_FALSE));
properties.setProperty(ORG_QUARTZ_THREADPOOL_CLASS, conf.getString(ORG_QUARTZ_THREADPOOL_CLASS, SimpleThreadPool.class.getName()));
properties.setProperty(ORG_QUARTZ_THREADPOOL_MAKETHREADSDAEMONS, conf.getString(ORG_QUARTZ_THREADPOOL_MAKETHREADSDAEMONS, STRING_TRUE));
properties.setProperty(ORG_QUARTZ_THREADPOOL_THREADCOUNT, conf.getString(ORG_QUARTZ_THREADPOOL_THREADCOUNT, QUARTZ_THREADCOUNT));
properties.setProperty(ORG_QUARTZ_THREADPOOL_THREADPRIORITY, conf.getString(ORG_QUARTZ_THREADPOOL_THREADPRIORITY, QUARTZ_THREADPRIORITY));
properties.setProperty(ORG_QUARTZ_JOBSTORE_CLASS, conf.getString(ORG_QUARTZ_JOBSTORE_CLASS, JobStoreTX.class.getName()));
properties.setProperty(ORG_QUARTZ_JOBSTORE_TABLEPREFIX, conf.getString(ORG_QUARTZ_JOBSTORE_TABLEPREFIX, QUARTZ_TABLE_PREFIX));
properties.setProperty(ORG_QUARTZ_JOBSTORE_ISCLUSTERED, conf.getString(ORG_QUARTZ_JOBSTORE_ISCLUSTERED, STRING_TRUE));
properties.setProperty(ORG_QUARTZ_JOBSTORE_MISFIRETHRESHOLD, conf.getString(ORG_QUARTZ_JOBSTORE_MISFIRETHRESHOLD, QUARTZ_MISFIRETHRESHOLD));
properties.setProperty(ORG_QUARTZ_JOBSTORE_CLUSTERCHECKININTERVAL, conf.getString(ORG_QUARTZ_JOBSTORE_CLUSTERCHECKININTERVAL, QUARTZ_CLUSTERCHECKININTERVAL));
properties.setProperty(ORG_QUARTZ_JOBSTORE_ACQUIRETRIGGERSWITHINLOCK, conf.getString(ORG_QUARTZ_JOBSTORE_ACQUIRETRIGGERSWITHINLOCK, QUARTZ_ACQUIRETRIGGERSWITHINLOCK));
properties.setProperty(ORG_QUARTZ_JOBSTORE_DATASOURCE, conf.getString(ORG_QUARTZ_JOBSTORE_DATASOURCE, QUARTZ_DATASOURCE));
properties.setProperty(ORG_QUARTZ_DATASOURCE_MYDS_CONNECTIONPROVIDER_CLASS, conf.getString(ORG_QUARTZ_DATASOURCE_MYDS_CONNECTIONPROVIDER_CLASS, DruidConnectionProvider.class.getName()));
schedulerFactory.initialize(properties);
scheduler = schedulerFactory.getScheduler();
} catch (SchedulerException e) {
logger.error(e.getMessage(), e);
System.exit(1);
}
} }
}
/**
* Whether the scheduler has been started.
/** *
* add task trigger , if this task already exists, return this task with updated trigger * @throws SchedulerException scheduler exception
* */
* @param clazz job class name public void start() throws SchedulerException {
* @param jobName job name if (!scheduler.isStarted()) {
* @param jobGroupName job group name scheduler.start();
* @param startDate job start date logger.info("Quartz service started");
* @param endDate job end date
* @param cronExpression cron expression
* @param jobDataMap job parameters data map
*/
public void addJob(Class<? extends Job> clazz,String jobName,String jobGroupName,Date startDate, Date endDate,
String cronExpression,
Map<String, Object> jobDataMap) {
lock.writeLock().lock();
try {
JobKey jobKey = new JobKey(jobName, jobGroupName);
JobDetail jobDetail;
//add a task (if this task already exists, return this task directly)
if (scheduler.checkExists(jobKey)) {
jobDetail = scheduler.getJobDetail(jobKey);
if (jobDataMap != null) {
jobDetail.getJobDataMap().putAll(jobDataMap);
} }
} else { }
jobDetail = newJob(clazz).withIdentity(jobKey).build();
if (jobDataMap != null) { /**
jobDetail.getJobDataMap().putAll(jobDataMap); * stop all scheduled tasks
* <p>
* Halts the Scheduler's firing of Triggers,
* and cleans up all resources associated with the Scheduler.
* <p>
* The scheduler cannot be re-started.
*
* @throws SchedulerException scheduler exception
*/
public void shutdown() throws SchedulerException {
if (!scheduler.isShutdown()) {
// don't wait for the task to complete
scheduler.shutdown();
logger.info("Quartz service stopped, and halt all tasks");
} }
}
scheduler.addJob(jobDetail, false, true); /**
* add task trigger , if this task already exists, return this task with updated trigger
logger.info("Add job, job name: {}, group name: {}", *
jobName, jobGroupName); * @param clazz job class name
} * @param jobName job name
* @param jobGroupName job group name
TriggerKey triggerKey = new TriggerKey(jobName, jobGroupName); * @param startDate job start date
/** * @param endDate job end date
* Instructs the Scheduler that upon a mis-fire * @param cronExpression cron expression
* situation, the CronTrigger wants to have it's * @param jobDataMap job parameters data map
* next-fire-time updated to the next time in the schedule after the */
* current time (taking into account any associated Calendar), public void addJob(Class<? extends Job> clazz, String jobName, String jobGroupName, Date startDate, Date endDate,
* but it does not want to be fired now. String cronExpression,
*/ Map<String, Object> jobDataMap) {
CronTrigger cronTrigger = newTrigger().withIdentity(triggerKey).startAt(startDate).endAt(endDate) lock.writeLock().lock();
.withSchedule(cronSchedule(cronExpression).withMisfireHandlingInstructionDoNothing()) try {
.forJob(jobDetail).build();
JobKey jobKey = new JobKey(jobName, jobGroupName);
if (scheduler.checkExists(triggerKey)) { JobDetail jobDetail;
// updateProcessInstance scheduler trigger when scheduler cycle changes //add a task (if this task already exists, return this task directly)
CronTrigger oldCronTrigger = (CronTrigger) scheduler.getTrigger(triggerKey); if (scheduler.checkExists(jobKey)) {
String oldCronExpression = oldCronTrigger.getCronExpression();
jobDetail = scheduler.getJobDetail(jobKey);
if (!StringUtils.equalsIgnoreCase(cronExpression,oldCronExpression)) { if (jobDataMap != null) {
// reschedule job trigger jobDetail.getJobDataMap().putAll(jobDataMap);
scheduler.rescheduleJob(triggerKey, cronTrigger); }
logger.info("reschedule job trigger, triggerName: {}, triggerGroupName: {}, cronExpression: {}, startDate: {}, endDate: {}", } else {
jobName, jobGroupName, cronExpression, startDate, endDate); jobDetail = newJob(clazz).withIdentity(jobKey).build();
}
} else { if (jobDataMap != null) {
scheduler.scheduleJob(cronTrigger); jobDetail.getJobDataMap().putAll(jobDataMap);
logger.info("schedule job trigger, triggerName: {}, triggerGroupName: {}, cronExpression: {}, startDate: {}, endDate: {}", }
jobName, jobGroupName, cronExpression, startDate, endDate);
} scheduler.addJob(jobDetail, false, true);
} catch (Exception e) { logger.info("Add job, job name: {}, group name: {}",
logger.error("add job failed", e); jobName, jobGroupName);
throw new RuntimeException("add job failed", e); }
} finally {
lock.writeLock().unlock(); TriggerKey triggerKey = new TriggerKey(jobName, jobGroupName);
/**
* Instructs the Scheduler that upon a mis-fire
* situation, the CronTrigger wants to have it's
* next-fire-time updated to the next time in the schedule after the
* current time (taking into account any associated Calendar),
* but it does not want to be fired now.
*/
CronTrigger cronTrigger = newTrigger().withIdentity(triggerKey).startAt(startDate).endAt(endDate)
.withSchedule(cronSchedule(cronExpression).withMisfireHandlingInstructionDoNothing())
.forJob(jobDetail).build();
if (scheduler.checkExists(triggerKey)) {
// updateProcessInstance scheduler trigger when scheduler cycle changes
CronTrigger oldCronTrigger = (CronTrigger) scheduler.getTrigger(triggerKey);
String oldCronExpression = oldCronTrigger.getCronExpression();
if (!StringUtils.equalsIgnoreCase(cronExpression, oldCronExpression)) {
// reschedule job trigger
scheduler.rescheduleJob(triggerKey, cronTrigger);
logger.info("reschedule job trigger, triggerName: {}, triggerGroupName: {}, cronExpression: {}, startDate: {}, endDate: {}",
jobName, jobGroupName, cronExpression, startDate, endDate);
}
} else {
scheduler.scheduleJob(cronTrigger);
logger.info("schedule job trigger, triggerName: {}, triggerGroupName: {}, cronExpression: {}, startDate: {}, endDate: {}",
jobName, jobGroupName, cronExpression, startDate, endDate);
}
} catch (Exception e) {
throw new ServiceException("add job failed", e);
} finally {
lock.writeLock().unlock();
}
} }
}
/**
* delete job
/** *
* delete job * @param jobName job name
* * @param jobGroupName job group name
* @param jobName job name * @return true if the Job was found and deleted.
* @param jobGroupName job group name */
* @return true if the Job was found and deleted. public boolean deleteJob(String jobName, String jobGroupName) {
*/ lock.writeLock().lock();
public boolean deleteJob(String jobName, String jobGroupName) { try {
lock.writeLock().lock(); JobKey jobKey = new JobKey(jobName, jobGroupName);
try { if (scheduler.checkExists(jobKey)) {
JobKey jobKey = new JobKey(jobName,jobGroupName); logger.info("try to delete job, job name: {}, job group name: {},", jobName, jobGroupName);
if(scheduler.checkExists(jobKey)){ return scheduler.deleteJob(jobKey);
logger.info("try to delete job, job name: {}, job group name: {},", jobName, jobGroupName); } else {
return scheduler.deleteJob(jobKey); return true;
}else { }
return true;
} } catch (SchedulerException e) {
logger.error("delete job : {} failed", jobName, e);
} catch (SchedulerException e) { } finally {
logger.error("delete job : {} failed",jobName, e); lock.writeLock().unlock();
} finally { }
lock.writeLock().unlock(); return false;
}
/**
* delete all jobs in job group
*
* @param jobGroupName job group name
* @return true if all of the Jobs were found and deleted, false if
* one or more were not deleted.
*/
public boolean deleteAllJobs(String jobGroupName) {
lock.writeLock().lock();
try {
logger.info("try to delete all jobs in job group: {}", jobGroupName);
List<JobKey> jobKeys = new ArrayList<>();
jobKeys.addAll(scheduler.getJobKeys(GroupMatcher.groupEndsWith(jobGroupName)));
return scheduler.deleteJobs(jobKeys);
} catch (SchedulerException e) {
logger.error("delete all jobs in job group: {} failed", jobGroupName, e);
} finally {
lock.writeLock().unlock();
}
return false;
} }
return false;
} /**
* build job name
/** *
* delete all jobs in job group * @param processId process id
* * @return job name
* @param jobGroupName job group name */
* public static String buildJobName(int processId) {
* @return true if all of the Jobs were found and deleted, false if StringBuilder sb = new StringBuilder(30);
* one or more were not deleted. sb.append(QUARTZ_JOB_PRIFIX).append(UNDERLINE).append(processId);
*/ return sb.toString();
public boolean deleteAllJobs(String jobGroupName) { }
lock.writeLock().lock();
try { /**
logger.info("try to delete all jobs in job group: {}", jobGroupName); * build job group name
List<JobKey> jobKeys = new ArrayList<>(); *
jobKeys.addAll(scheduler.getJobKeys(GroupMatcher.groupEndsWith(jobGroupName))); * @param projectId project id
* @return job group name
return scheduler.deleteJobs(jobKeys); */
} catch (SchedulerException e) { public static String buildJobGroupName(int projectId) {
logger.error("delete all jobs in job group: {} failed",jobGroupName, e); StringBuilder sb = new StringBuilder(30);
} finally { sb.append(QUARTZ_JOB_GROUP_PRIFIX).append(UNDERLINE).append(projectId);
lock.writeLock().unlock(); return sb.toString();
}
/**
* add params to map
*
* @param projectId project id
* @param scheduleId schedule id
* @param schedule schedule
* @return data map
*/
public static Map<String, Object> buildDataMap(int projectId, int scheduleId, Schedule schedule) {
Map<String, Object> dataMap = new HashMap<>(3);
dataMap.put(PROJECT_ID, projectId);
dataMap.put(SCHEDULE_ID, scheduleId);
dataMap.put(SCHEDULE, JSONUtils.toJsonString(schedule));
return dataMap;
} }
return false;
}
/**
* build job name
* @param processId process id
* @return job name
*/
public static String buildJobName(int processId) {
StringBuilder sb = new StringBuilder(30);
sb.append(QUARTZ_JOB_PRIFIX).append(UNDERLINE).append(processId);
return sb.toString();
}
/**
* build job group name
* @param projectId project id
* @return job group name
*/
public static String buildJobGroupName(int projectId) {
StringBuilder sb = new StringBuilder(30);
sb.append(QUARTZ_JOB_GROUP_PRIFIX).append(UNDERLINE).append(projectId);
return sb.toString();
}
/**
* add params to map
*
* @param projectId project id
* @param scheduleId schedule id
* @param schedule schedule
* @return data map
*/
public static Map<String, Object> buildDataMap(int projectId, int scheduleId, Schedule schedule) {
Map<String, Object> dataMap = new HashMap<>(3);
dataMap.put(PROJECT_ID, projectId);
dataMap.put(SCHEDULE_ID, scheduleId);
dataMap.put(SCHEDULE, JSONUtils.toJsonString(schedule));
return dataMap;
}
} }

302
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/AbstractCycle.java
Unescape View File

@ -14,159 +14,177 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.service.quartz.cron; package org.apache.dolphinscheduler.service.quartz.cron;
import org.apache.dolphinscheduler.common.enums.CycleEnum;
import com.cronutils.model.Cron; import com.cronutils.model.Cron;
import com.cronutils.model.field.CronField; import com.cronutils.model.field.CronField;
import com.cronutils.model.field.CronFieldName; import com.cronutils.model.field.CronFieldName;
import com.cronutils.model.field.expression.*; import com.cronutils.model.field.expression.Always;
import org.apache.dolphinscheduler.common.enums.CycleEnum; import com.cronutils.model.field.expression.And;
import com.cronutils.model.field.expression.Between;
import com.cronutils.model.field.expression.Every;
import com.cronutils.model.field.expression.FieldExpression;
import com.cronutils.model.field.expression.On;
/** /**
* Cycle * Cycle
*/ */
public abstract class AbstractCycle { public abstract class AbstractCycle {
protected Cron cron; protected Cron cron;
protected CronField minField; protected CronField minField;
protected CronField hourField; protected CronField hourField;
protected CronField dayOfMonthField; protected CronField dayOfMonthField;
protected CronField dayOfWeekField; protected CronField dayOfWeekField;
protected CronField monthField; protected CronField monthField;
protected CronField yearField; protected CronField yearField;
public CycleLinks addCycle(AbstractCycle cycle) { public CycleLinks addCycle(AbstractCycle cycle) {
return new CycleLinks(this.cron).addCycle(this).addCycle(cycle); return new CycleLinks(this.cron).addCycle(this).addCycle(cycle);
} }
/** /**
* cycle constructor * cycle constructor
* @param cron cron *
*/ * @param cron cron
public AbstractCycle(Cron cron) { */
if (cron == null) { protected AbstractCycle(Cron cron) {
throw new IllegalArgumentException("cron must not be null!"); if (cron == null) {
throw new IllegalArgumentException("cron must not be null!");
}
this.cron = cron;
this.minField = cron.retrieve(CronFieldName.MINUTE);
this.hourField = cron.retrieve(CronFieldName.HOUR);
this.dayOfMonthField = cron.retrieve(CronFieldName.DAY_OF_MONTH);
this.dayOfWeekField = cron.retrieve(CronFieldName.DAY_OF_WEEK);
this.monthField = cron.retrieve(CronFieldName.MONTH);
this.yearField = cron.retrieve(CronFieldName.YEAR);
}
/**
* whether the minute field has a value
*
* @return if minute field has a value return true，else return false
*/
protected boolean minFiledIsSetAll() {
FieldExpression minFieldExpression = minField.getExpression();
return (minFieldExpression instanceof Every || minFieldExpression instanceof Always
|| minFieldExpression instanceof Between || minFieldExpression instanceof And
|| minFieldExpression instanceof On);
}
/**
* whether the minute field has a value of every or always
*
* @return if minute field has a value of every or always return true，else return false
*/
protected boolean minFiledIsEvery() {
FieldExpression minFieldExpression = minField.getExpression();
return (minFieldExpression instanceof Every || minFieldExpression instanceof Always);
}
/**
* whether the hour field has a value
*
* @return if hour field has a value return true，else return false
*/
protected boolean hourFiledIsSetAll() {
FieldExpression hourFieldExpression = hourField.getExpression();
return (hourFieldExpression instanceof Every || hourFieldExpression instanceof Always
|| hourFieldExpression instanceof Between || hourFieldExpression instanceof And
|| hourFieldExpression instanceof On);
}
/**
* whether the hour field has a value of every or always
*
* @return if hour field has a value of every or always return true，else return false
*/
protected boolean hourFiledIsEvery() {
FieldExpression hourFieldExpression = hourField.getExpression();
return (hourFieldExpression instanceof Every || hourFieldExpression instanceof Always);
}
/**
* whether the day Of month field has a value
*
* @return if day Of month field has a value return true，else return false
*/
protected boolean dayOfMonthFieldIsSetAll() {
return (dayOfMonthField.getExpression() instanceof Every || dayOfMonthField.getExpression() instanceof Always
|| dayOfMonthField.getExpression() instanceof Between || dayOfMonthField.getExpression() instanceof And
|| dayOfMonthField.getExpression() instanceof On);
}
/**
* whether the day Of Month field has a value of every or always
*
* @return if day Of Month field has a value of every or always return true，else return false
*/
protected boolean dayOfMonthFieldIsEvery() {
return (dayOfMonthField.getExpression() instanceof Every || dayOfMonthField.getExpression() instanceof Always);
}
/**
* whether month field has a value
*
* @return if month field has a value return true，else return false
*/
protected boolean monthFieldIsSetAll() {
FieldExpression monthFieldExpression = monthField.getExpression();
return (monthFieldExpression instanceof Every || monthFieldExpression instanceof Always
|| monthFieldExpression instanceof Between || monthFieldExpression instanceof And
|| monthFieldExpression instanceof On);
}
/**
* whether the month field has a value of every or always
*
* @return if month field has a value of every or always return true，else return false
*/
protected boolean monthFieldIsEvery() {
FieldExpression monthFieldExpression = monthField.getExpression();
return (monthFieldExpression instanceof Every || monthFieldExpression instanceof Always);
}
/**
* whether the day Of week field has a value
*
* @return if day Of week field has a value return true，else return false
*/
protected boolean dayofWeekFieldIsSetAll() {
FieldExpression dayOfWeekFieldExpression = dayOfWeekField.getExpression();
return (dayOfWeekFieldExpression instanceof Every || dayOfWeekFieldExpression instanceof Always
|| dayOfWeekFieldExpression instanceof Between || dayOfWeekFieldExpression instanceof And
|| dayOfWeekFieldExpression instanceof On);
}
/**
* whether the day Of week field has a value of every or always
*
* @return if day Of week field has a value of every or always return true，else return false
*/
protected boolean dayofWeekFieldIsEvery() {
FieldExpression dayOfWeekFieldExpression = dayOfWeekField.getExpression();
return (dayOfWeekFieldExpression instanceof Every || dayOfWeekFieldExpression instanceof Always);
} }
this.cron = cron; /**
this.minField = cron.retrieve(CronFieldName.MINUTE); * get cycle enum
this.hourField = cron.retrieve(CronFieldName.HOUR); *
this.dayOfMonthField = cron.retrieve(CronFieldName.DAY_OF_MONTH); * @return CycleEnum
this.dayOfWeekField = cron.retrieve(CronFieldName.DAY_OF_WEEK); */
this.monthField = cron.retrieve(CronFieldName.MONTH); protected abstract CycleEnum getCycle();
this.yearField = cron.retrieve(CronFieldName.YEAR);
} /**
* get mini level cycle enum
/** *
* whether the minute field has a value * @return CycleEnum
* @return if minute field has a value return true，else return false */
*/ protected abstract CycleEnum getMiniCycle();
protected boolean minFiledIsSetAll(){
FieldExpression minFieldExpression = minField.getExpression();
return (minFieldExpression instanceof Every || minFieldExpression instanceof Always
|| minFieldExpression instanceof Between || minFieldExpression instanceof And
|| minFieldExpression instanceof On);
}
/**
* whether the minute field has a value of every or always
* @return if minute field has a value of every or always return true，else return false
*/
protected boolean minFiledIsEvery(){
FieldExpression minFieldExpression = minField.getExpression();
return (minFieldExpression instanceof Every || minFieldExpression instanceof Always);
}
/**
* whether the hour field has a value
* @return if hour field has a value return true，else return false
*/
protected boolean hourFiledIsSetAll(){
FieldExpression hourFieldExpression = hourField.getExpression();
return (hourFieldExpression instanceof Every || hourFieldExpression instanceof Always
|| hourFieldExpression instanceof Between || hourFieldExpression instanceof And
|| hourFieldExpression instanceof On);
}
/**
* whether the hour field has a value of every or always
* @return if hour field has a value of every or always return true，else return false
*/
protected boolean hourFiledIsEvery(){
FieldExpression hourFieldExpression = hourField.getExpression();
return (hourFieldExpression instanceof Every || hourFieldExpression instanceof Always);
}
/**
* whether the day Of month field has a value
* @return if day Of month field has a value return true，else return false
*/
protected boolean dayOfMonthFieldIsSetAll(){
return (dayOfMonthField.getExpression() instanceof Every || dayOfMonthField.getExpression() instanceof Always
|| dayOfMonthField.getExpression() instanceof Between || dayOfMonthField.getExpression() instanceof And
|| dayOfMonthField.getExpression() instanceof On);
}
/**
* whether the day Of Month field has a value of every or always
* @return if day Of Month field has a value of every or always return true，else return false
*/
protected boolean dayOfMonthFieldIsEvery(){
return (dayOfMonthField.getExpression() instanceof Every || dayOfMonthField.getExpression() instanceof Always);
}
/**
* whether month field has a value
* @return if month field has a value return true，else return false
*/
protected boolean monthFieldIsSetAll(){
FieldExpression monthFieldExpression = monthField.getExpression();
return (monthFieldExpression instanceof Every || monthFieldExpression instanceof Always
|| monthFieldExpression instanceof Between || monthFieldExpression instanceof And
|| monthFieldExpression instanceof On);
}
/**
* whether the month field has a value of every or always
* @return if month field has a value of every or always return true，else return false
*/
protected boolean monthFieldIsEvery(){
FieldExpression monthFieldExpression = monthField.getExpression();
return (monthFieldExpression instanceof Every || monthFieldExpression instanceof Always);
}
/**
* whether the day Of week field has a value
* @return if day Of week field has a value return true，else return false
*/
protected boolean dayofWeekFieldIsSetAll(){
FieldExpression dayOfWeekFieldExpression = dayOfWeekField.getExpression();
return (dayOfWeekFieldExpression instanceof Every || dayOfWeekFieldExpression instanceof Always
|| dayOfWeekFieldExpression instanceof Between || dayOfWeekFieldExpression instanceof And
|| dayOfWeekFieldExpression instanceof On);
}
/**
* whether the day Of week field has a value of every or always
* @return if day Of week field has a value of every or always return true，else return false
*/
protected boolean dayofWeekFieldIsEvery(){
FieldExpression dayOfWeekFieldExpression = dayOfWeekField.getExpression();
return (dayOfWeekFieldExpression instanceof Every || dayOfWeekFieldExpression instanceof Always);
}
/**
* get cycle enum
* @return CycleEnum
*/
protected abstract CycleEnum getCycle();
/**
* get mini level cycle enum
* @return CycleEnum
*/
protected abstract CycleEnum getMiniCycle();
} }

611
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java
Unescape View File

@ -14,322 +14,329 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.service.zk; package org.apache.dolphinscheduler.service.zk;
import org.apache.curator.framework.recipes.locks.InterProcessMutex; import static org.apache.dolphinscheduler.common.Constants.ADD_ZK_OP;
import static org.apache.dolphinscheduler.common.Constants.COLON;
import static org.apache.dolphinscheduler.common.Constants.DELETE_ZK_OP;
import static org.apache.dolphinscheduler.common.Constants.DIVISION_STRING;
import static org.apache.dolphinscheduler.common.Constants.MASTER_PREFIX;
import static org.apache.dolphinscheduler.common.Constants.SINGLE_SLASH;
import static org.apache.dolphinscheduler.common.Constants.UNDERLINE;
import static org.apache.dolphinscheduler.common.Constants.WORKER_PREFIX;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ZKNodeType; import org.apache.dolphinscheduler.common.enums.ZKNodeType;
import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.utils.ResInfo; import org.apache.dolphinscheduler.common.utils.ResInfo;
import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.util.*;
import static org.apache.dolphinscheduler.common.Constants.*;
/** /**
* abstract zookeeper client * abstract zookeeper client
*/ */
@Component @Component
public abstract class AbstractZKClient extends ZookeeperCachedOperator { public abstract class AbstractZKClient extends ZookeeperCachedOperator {
private static final Logger logger = LoggerFactory.getLogger(AbstractZKClient.class); private static final Logger logger = LoggerFactory.getLogger(AbstractZKClient.class);
/**
/** * remove dead server by host
* remove dead server by host *
* @param host host * @param host host
* @param serverType serverType * @param serverType serverType
* @throws Exception */
*/ public void removeDeadServerByHost(String host, String serverType) {
public void removeDeadServerByHost(String host, String serverType) throws Exception { List<String> deadServers = super.getChildrenKeys(getDeadZNodeParentPath());
List<String> deadServers = super.getChildrenKeys(getDeadZNodeParentPath()); for (String serverPath : deadServers) {
for(String serverPath : deadServers){ if (serverPath.startsWith(serverType + UNDERLINE + host)) {
if(serverPath.startsWith(serverType+UNDERLINE+host)){ String server = getDeadZNodeParentPath() + SINGLE_SLASH + serverPath;
String server = getDeadZNodeParentPath() + SINGLE_SLASH + serverPath; super.remove(server);
super.remove(server); logger.info("{} server {} deleted from zk dead server path success", serverType, host);
logger.info("{} server {} deleted from zk dead server path success" , serverType , host); }
} }
} }
}
/**
* opType(add): if find dead server , then add to zk deadServerPath
/** * opType(delete): delete path from zk
* opType(add): if find dead server , then add to zk deadServerPath *
* opType(delete): delete path from zk * @param zNode node path
* * @param zkNodeType master or worker
* @param zNode node path * @param opType delete or add
* @param zkNodeType master or worker */
* @param opType delete or add public void handleDeadServer(String zNode, ZKNodeType zkNodeType, String opType) {
* @throws Exception errors String host = getHostByEventDataPath(zNode);
*/ String type = (zkNodeType == ZKNodeType.MASTER) ? MASTER_PREFIX : WORKER_PREFIX;
public void handleDeadServer(String zNode, ZKNodeType zkNodeType, String opType) throws Exception {
String host = getHostByEventDataPath(zNode); //check server restart, if restart , dead server path in zk should be delete
String type = (zkNodeType == ZKNodeType.MASTER) ? MASTER_PREFIX : WORKER_PREFIX; if (opType.equals(DELETE_ZK_OP)) {
removeDeadServerByHost(host, type);
//check server restart, if restart , dead server path in zk should be delete
if(opType.equals(DELETE_ZK_OP)){ } else if (opType.equals(ADD_ZK_OP)) {
removeDeadServerByHost(host, type); String deadServerPath = getDeadZNodeParentPath() + SINGLE_SLASH + type + UNDERLINE + host;
if (!super.isExisted(deadServerPath)) {
}else if(opType.equals(ADD_ZK_OP)){ //add dead server info to zk dead server path : /dead-servers/
String deadServerPath = getDeadZNodeParentPath() + SINGLE_SLASH + type + UNDERLINE + host;
if(!super.isExisted(deadServerPath)){ super.persist(deadServerPath, (type + UNDERLINE + host));
//add dead server info to zk dead server path : /dead-servers/
logger.info("{} server dead , and {} added to zk dead server path success",
super.persist(deadServerPath,(type + UNDERLINE + host)); zkNodeType, zNode);
}
logger.info("{} server dead , and {} added to zk dead server path success" , }
zkNodeType.toString(), zNode);
} }
}
/**
} * get active master num
*
/** * @return active master number
* get active master num */
* @return active master number public int getActiveMasterNum() {
*/ List<String> childrenList = new ArrayList<>();
public int getActiveMasterNum(){ try {
List<String> childrenList = new ArrayList<>(); // read master node parent path from conf
try { if (super.isExisted(getZNodeParentPath(ZKNodeType.MASTER))) {
// read master node parent path from conf childrenList = super.getChildrenKeys(getZNodeParentPath(ZKNodeType.MASTER));
if(super.isExisted(getZNodeParentPath(ZKNodeType.MASTER))){ }
childrenList = super.getChildrenKeys(getZNodeParentPath(ZKNodeType.MASTER)); } catch (Exception e) {
} logger.error("getActiveMasterNum error", e);
} catch (Exception e) { }
logger.error("getActiveMasterNum error",e); return childrenList.size();
} }
return childrenList.size();
} /**
* @return zookeeper quorum
/** */
* public String getZookeeperQuorum() {
* @return zookeeper quorum return getZookeeperConfig().getServerList();
*/ }
public String getZookeeperQuorum(){
return getZookeeperConfig().getServerList(); /**
} * get server list.
*
/** * @param zkNodeType zookeeper node type
* get server list. * @return server list
* @param zkNodeType zookeeper node type */
* @return server list public List<Server> getServersList(ZKNodeType zkNodeType) {
*/ Map<String, String> masterMap = getServerMaps(zkNodeType);
public List<Server> getServersList(ZKNodeType zkNodeType){ String parentPath = getZNodeParentPath(zkNodeType);
Map<String, String> masterMap = getServerMaps(zkNodeType);
String parentPath = getZNodeParentPath(zkNodeType); List<Server> masterServers = new ArrayList<>();
for (Map.Entry<String, String> entry : masterMap.entrySet()) {
List<Server> masterServers = new ArrayList<>(); Server masterServer = ResInfo.parseHeartbeatForZKInfo(entry.getValue());
for (Map.Entry<String, String> entry : masterMap.entrySet()) { if (masterServer == null) {
Server masterServer = ResInfo.parseHeartbeatForZKInfo(entry.getValue()); continue;
if(masterServer == null){ }
continue; String key = entry.getKey();
} masterServer.setZkDirectory(parentPath + "/" + key);
String key = entry.getKey(); //set host and port
masterServer.setZkDirectory(parentPath + "/"+ key); String[] hostAndPort = key.split(COLON);
//set host and port String[] hosts = hostAndPort[0].split(DIVISION_STRING);
String[] hostAndPort=key.split(COLON); // fetch the last one
String[] hosts=hostAndPort[0].split(DIVISION_STRING); masterServer.setHost(hosts[hosts.length - 1]);
// fetch the last one masterServer.setPort(Integer.parseInt(hostAndPort[1]));
masterServer.setHost(hosts[hosts.length-1]); masterServers.add(masterServer);
masterServer.setPort(Integer.parseInt(hostAndPort[1])); }
masterServers.add(masterServer); return masterServers;
} }
return masterServers;
} /**
* get master server list map.
/** *
* get master server list map. * @param zkNodeType zookeeper node type
* @param zkNodeType zookeeper node type * @return result : {host : resource info}
* @return result : {host : resource info} */
*/ public Map<String, String> getServerMaps(ZKNodeType zkNodeType) {
public Map<String, String> getServerMaps(ZKNodeType zkNodeType){
Map<String, String> masterMap = new HashMap<>();
Map<String, String> masterMap = new HashMap<>(); try {
try { String path = getZNodeParentPath(zkNodeType);
String path = getZNodeParentPath(zkNodeType); List<String> serverList = super.getChildrenKeys(path);
List<String> serverList = super.getChildrenKeys(path); if (zkNodeType == ZKNodeType.WORKER) {
if(zkNodeType == ZKNodeType.WORKER){ List<String> workerList = new ArrayList<>();
List<String> workerList = new ArrayList<>(); for (String group : serverList) {
for(String group : serverList){ List<String> groupServers = super.getChildrenKeys(path + Constants.SLASH + group);
List<String> groupServers = super.getChildrenKeys(path + Constants.SLASH + group); for (String groupServer : groupServers) {
for(String groupServer : groupServers){ workerList.add(group + Constants.SLASH + groupServer);
workerList.add(group + Constants.SLASH + groupServer); }
} }
} serverList = workerList;
serverList = workerList; }
} for (String server : serverList) {
for(String server : serverList){ masterMap.putIfAbsent(server, super.get(path + Constants.SLASH + server));
masterMap.putIfAbsent(server, super.get(path + Constants.SLASH + server)); }
} } catch (Exception e) {
} catch (Exception e) { logger.error("get server list failed", e);
logger.error("get server list failed", e); }
}
return masterMap;
return masterMap; }
}
/**
/** * check the zookeeper node already exists
* check the zookeeper node already exists *
* @param host host * @param host host
* @param zkNodeType zookeeper node type * @param zkNodeType zookeeper node type
* @return true if exists * @return true if exists
*/ */
public boolean checkZKNodeExists(String host, ZKNodeType zkNodeType) { public boolean checkZKNodeExists(String host, ZKNodeType zkNodeType) {
String path = getZNodeParentPath(zkNodeType); String path = getZNodeParentPath(zkNodeType);
if(StringUtils.isEmpty(path)){ if (StringUtils.isEmpty(path)) {
logger.error("check zk node exists error, host:{}, zk node type:{}", logger.error("check zk node exists error, host:{}, zk node type:{}",
host, zkNodeType.toString()); host, zkNodeType);
return false; return false;
} }
Map<String, String> serverMaps = getServerMaps(zkNodeType); Map<String, String> serverMaps = getServerMaps(zkNodeType);
for(String hostKey : serverMaps.keySet()){ for (String hostKey : serverMaps.keySet()) {
if(hostKey.contains(host)){ if (hostKey.contains(host)) {
return true; return true;
} }
} }
return false; return false;
} }
/** /**
* * @return get worker node parent path
* @return get worker node parent path */
*/ protected String getWorkerZNodeParentPath() {
protected String getWorkerZNodeParentPath(){ return getZookeeperConfig().getDsRoot() + Constants.ZOOKEEPER_DOLPHINSCHEDULER_WORKERS;
return getZookeeperConfig().getDsRoot() + Constants.ZOOKEEPER_DOLPHINSCHEDULER_WORKERS; }
}
/**
/** * @return get master node parent path
* */
* @return get master node parent path protected String getMasterZNodeParentPath() {
*/ return getZookeeperConfig().getDsRoot() + Constants.ZOOKEEPER_DOLPHINSCHEDULER_MASTERS;
protected String getMasterZNodeParentPath(){ }
return getZookeeperConfig().getDsRoot() + Constants.ZOOKEEPER_DOLPHINSCHEDULER_MASTERS;
} /**
* @return get master lock path
/** */
* public String getMasterLockPath() {
* @return get master lock path return getZookeeperConfig().getDsRoot() + Constants.ZOOKEEPER_DOLPHINSCHEDULER_LOCK_MASTERS;
*/ }
public String getMasterLockPath(){
return getZookeeperConfig().getDsRoot() + Constants.ZOOKEEPER_DOLPHINSCHEDULER_LOCK_MASTERS; /**
} * @param zkNodeType zookeeper node type
* @return get zookeeper node parent path
/** */
* public String getZNodeParentPath(ZKNodeType zkNodeType) {
* @param zkNodeType zookeeper node type String path = "";
* @return get zookeeper node parent path switch (zkNodeType) {
*/ case MASTER:
public String getZNodeParentPath(ZKNodeType zkNodeType) { return getMasterZNodeParentPath();
String path = ""; case WORKER:
switch (zkNodeType){ return getWorkerZNodeParentPath();
case MASTER: case DEAD_SERVER:
return getMasterZNodeParentPath(); return getDeadZNodeParentPath();
case WORKER: default:
return getWorkerZNodeParentPath(); break;
case DEAD_SERVER: }
return getDeadZNodeParentPath(); return path;
default: }
break;
} /**
return path; * @return get dead server node parent path
} */
protected String getDeadZNodeParentPath() {
/** return getZookeeperConfig().getDsRoot() + Constants.ZOOKEEPER_DOLPHINSCHEDULER_DEAD_SERVERS;
* }
* @return get dead server node parent path
*/ /**
protected String getDeadZNodeParentPath(){ * @return get master start up lock path
return getZookeeperConfig().getDsRoot() + Constants.ZOOKEEPER_DOLPHINSCHEDULER_DEAD_SERVERS; */
} public String getMasterStartUpLockPath() {
return getZookeeperConfig().getDsRoot() + Constants.ZOOKEEPER_DOLPHINSCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS;
/** }
*
* @return get master start up lock path /**
*/ * @return get master failover lock path
public String getMasterStartUpLockPath(){ */
return getZookeeperConfig().getDsRoot() + Constants.ZOOKEEPER_DOLPHINSCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS; public String getMasterFailoverLockPath() {
} return getZookeeperConfig().getDsRoot() + Constants.ZOOKEEPER_DOLPHINSCHEDULER_LOCK_FAILOVER_MASTERS;
}
/**
* /**
* @return get master failover lock path * @return get worker failover lock path
*/ */
public String getMasterFailoverLockPath(){ public String getWorkerFailoverLockPath() {
return getZookeeperConfig().getDsRoot() + Constants.ZOOKEEPER_DOLPHINSCHEDULER_LOCK_FAILOVER_MASTERS; return getZookeeperConfig().getDsRoot() + Constants.ZOOKEEPER_DOLPHINSCHEDULER_LOCK_FAILOVER_WORKERS;
} }
/** /**
* * release mutex
* @return get worker failover lock path *
*/ * @param mutex mutex
public String getWorkerFailoverLockPath(){ */
return getZookeeperConfig().getDsRoot() + Constants.ZOOKEEPER_DOLPHINSCHEDULER_LOCK_FAILOVER_WORKERS; public void releaseMutex(InterProcessMutex mutex) {
} if (mutex != null) {
try {
/** mutex.release();
* release mutex } catch (Exception e) {
* @param mutex mutex if ("instance must be started before calling this method".equals(e.getMessage())) {
*/ logger.warn("lock release");
public void releaseMutex(InterProcessMutex mutex) { } else {
if (mutex != null){ logger.error("lock release failed", e);
try { }
mutex.release();
} catch (Exception e) { }
if("instance must be started before calling this method".equals(e.getMessage())){ }
logger.warn("lock release"); }
}else{
logger.error("lock release failed",e); /**
} * init system znode
*/
} protected void initSystemZNode() {
} try {
} persist(getMasterZNodeParentPath(), "");
persist(getWorkerZNodeParentPath(), "");
/** persist(getDeadZNodeParentPath(), "");
* init system znode
*/ logger.info("initialize server nodes success.");
protected void initSystemZNode(){ } catch (Exception e) {
try { logger.error("init system znode failed", e);
persist(getMasterZNodeParentPath(), ""); }
persist(getWorkerZNodeParentPath(), ""); }
persist(getDeadZNodeParentPath(), "");
/**
logger.info("initialize server nodes success."); * get host ip, string format: masterParentPath/ip
} catch (Exception e) { *
logger.error("init system znode failed",e); * @param path path
} * @return host ip, string format: masterParentPath/ip
} */
protected String getHostByEventDataPath(String path) {
/** if (StringUtils.isEmpty(path)) {
* get host ip, string format: masterParentPath/ip logger.error("empty path!");
* @param path path return "";
* @return host ip, string format: masterParentPath/ip }
*/ String[] pathArray = path.split(SINGLE_SLASH);
protected String getHostByEventDataPath(String path) { if (pathArray.length < 1) {
if(StringUtils.isEmpty(path)){ logger.error("parse ip error: {}", path);
logger.error("empty path!"); return "";
return ""; }
} return pathArray[pathArray.length - 1];
String[] pathArray = path.split(SINGLE_SLASH);
if(pathArray.length < 1){ }
logger.error("parse ip error: {}", path);
return ""; @Override
} public String toString() {
return pathArray[pathArray.length - 1]; return "AbstractZKClient{"
+ "zkClient=" + getZkClient()
} + ", deadServerZNodeParentPath='" + getZNodeParentPath(ZKNodeType.DEAD_SERVER) + '\''
+ ", masterZNodeParentPath='" + getZNodeParentPath(ZKNodeType.MASTER) + '\''
@Override + ", workerZNodeParentPath='" + getZNodeParentPath(ZKNodeType.WORKER) + '\''
public String toString() { + '}';
return "AbstractZKClient{" + }
"zkClient=" + getZkClient() + }
", deadServerZNodeParentPath='" + getZNodeParentPath(ZKNodeType.DEAD_SERVER) + '\'' +
", masterZNodeParentPath='" + getZNodeParentPath(ZKNodeType.MASTER) + '\'' +
", workerZNodeParentPath='" + getZNodeParentPath(ZKNodeType.WORKER) + '\'' +
'}';
}
}

22
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClient.java
Unescape View File

@ -14,9 +14,14 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.service.zk; package org.apache.dolphinscheduler.service.zk;
import org.apache.commons.lang.StringUtils; import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.service.exceptions.ServiceException;
import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.api.ACLProvider; import org.apache.curator.framework.api.ACLProvider;
@ -25,18 +30,16 @@ import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.ACL;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.TimeUnit;
import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull;
/** /**
* Shared Curator zookeeper client * Shared Curator zookeeper client
*/ */
@ -49,7 +52,6 @@ public class CuratorZookeeperClient implements InitializingBean {
private CuratorFramework zkClient; private CuratorFramework zkClient;
@Override @Override
public void afterPropertiesSet() throws Exception { public void afterPropertiesSet() throws Exception {
this.zkClient = buildClient(); this.zkClient = buildClient();
@ -91,7 +93,7 @@ public class CuratorZookeeperClient implements InitializingBean {
zkClient.blockUntilConnected(30, TimeUnit.SECONDS); zkClient.blockUntilConnected(30, TimeUnit.SECONDS);
} catch (final Exception ex) { } catch (final Exception ex) {
throw new RuntimeException(ex); throw new ServiceException(ex);
} }
return zkClient; return zkClient;
} }
@ -123,4 +125,4 @@ public class CuratorZookeeperClient implements InitializingBean {
public CuratorFramework getZkClient() { public CuratorFramework getZkClient() {
return zkClient; return zkClient;
} }
} }

37
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java
Unescape View File

@ -14,19 +14,22 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.service.zk; package org.apache.dolphinscheduler.service.zk;
import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.service.exceptions.ServiceException;
import org.apache.zookeeper.server.ZooKeeperServer; import org.apache.zookeeper.server.ZooKeeperServer;
import org.apache.zookeeper.server.ZooKeeperServerMain; import org.apache.zookeeper.server.ZooKeeperServerMain;
import org.apache.zookeeper.server.quorum.QuorumPeerConfig; import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* just speed experience version * just speed experience version
@ -51,10 +54,10 @@ public class ZKServer {
ZKServer zkServer; ZKServer zkServer;
if (args.length == 0) { if (args.length == 0) {
zkServer = new ZKServer(); zkServer = new ZKServer();
} else if (args.length == 1){ } else if (args.length == 1) {
zkServer = new ZKServer(Integer.valueOf(args[0]), ""); zkServer = new ZKServer(Integer.parseInt(args[0]), "");
} else { } else {
zkServer = new ZKServer(Integer.valueOf(args[0]), args[1]); zkServer = new ZKServer(Integer.parseInt(args[0]), args[1]);
} }
zkServer.registerHook(); zkServer.registerHook();
zkServer.start(); zkServer.start();
@ -73,7 +76,7 @@ public class ZKServer {
} }
private void registerHook() { private void registerHook() {
/** /*
* register hooks, which are called before the process exits * register hooks, which are called before the process exits
*/ */
Runtime.getRuntime().addShutdownHook(new Thread(this::stop)); Runtime.getRuntime().addShutdownHook(new Thread(this::stop));
@ -90,7 +93,7 @@ public class ZKServer {
} }
} }
public boolean isStarted(){ public boolean isStarted() {
return isStarted.get(); return isStarted.get();
} }
@ -119,19 +122,19 @@ public class ZKServer {
if (file.exists()) { if (file.exists()) {
logger.warn("The path of zk server exists"); logger.warn("The path of zk server exists");
} }
logger.info("zk server starting, data dir path:{}" , zkDataDir); logger.info("zk server starting, data dir path:{}", zkDataDir);
startLocalZkServer(port, zkDataDir, ZooKeeperServer.DEFAULT_TICK_TIME,"60"); startLocalZkServer(port, zkDataDir, ZooKeeperServer.DEFAULT_TICK_TIME, "60");
} }
/** /**
* Starts a local Zk instance * Starts a local Zk instance
* *
* @param port The port to listen on * @param port The port to listen on
* @param dataDirPath The path for the Zk data directory * @param dataDirPath The path for the Zk data directory
* @param tickTime zk tick time * @param tickTime zk tick time
* @param maxClientCnxns zk max client connections * @param maxClientCnxns zk max client connections
*/ */
private void startLocalZkServer(final int port, final String dataDirPath,final int tickTime,String maxClientCnxns) { private void startLocalZkServer(final int port, final String dataDirPath, final int tickTime, String maxClientCnxns) {
if (isStarted.compareAndSet(false, true)) { if (isStarted.compareAndSet(false, true)) {
zooKeeperServerMain = new PublicZooKeeperServerMain(); zooKeeperServerMain = new PublicZooKeeperServerMain();
logger.info("Zookeeper data path : {} ", dataDirPath); logger.info("Zookeeper data path : {} ", dataDirPath);
@ -144,8 +147,7 @@ public class ZKServer {
zooKeeperServerMain.initializeAndRun(args); zooKeeperServerMain.initializeAndRun(args);
} catch (QuorumPeerConfig.ConfigException | IOException e) { } catch (QuorumPeerConfig.ConfigException | IOException e) {
logger.warn("Caught exception while starting ZK", e); throw new ServiceException("Caught exception while starting ZK", e);
throw new RuntimeException(e);
} }
} }
} }
@ -159,7 +161,7 @@ public class ZKServer {
logger.info("zk server stopped"); logger.info("zk server stopped");
} catch (Exception e) { } catch (Exception e) {
logger.error("Failed to stop ZK ",e); logger.error("Failed to stop ZK ", e);
} }
} }
@ -180,8 +182,7 @@ public class ZKServer {
org.apache.commons.io.FileUtils.deleteDirectory(new File(dataDir)); org.apache.commons.io.FileUtils.deleteDirectory(new File(dataDir));
} }
} catch (Exception e) { } catch (Exception e) {
logger.warn("Caught exception while stopping ZK server", e); throw new ServiceException("Caught exception while starting ZK", e);
throw new RuntimeException(e);
} }
} }
} }

20
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperCachedOperator.java
Unescape View File

@ -14,21 +14,24 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.service.zk; package org.apache.dolphinscheduler.service.zk;
import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.service.exceptions.ServiceException;
import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.ChildData;
import org.apache.curator.framework.recipes.cache.TreeCache; import org.apache.curator.framework.recipes.cache.TreeCache;
import org.apache.curator.framework.recipes.cache.TreeCacheEvent; import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
import org.apache.curator.framework.recipes.cache.TreeCacheListener; import org.apache.curator.framework.recipes.cache.TreeCacheListener;
import java.nio.charset.StandardCharsets;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.nio.charset.StandardCharsets;
@Component @Component
public class ZookeeperCachedOperator extends ZookeeperOperator { public class ZookeeperCachedOperator extends ZookeeperOperator {
@ -36,6 +39,7 @@ public class ZookeeperCachedOperator extends ZookeeperOperator {
private TreeCache treeCache; private TreeCache treeCache;
/** /**
* register a unified listener of /${dsRoot}, * register a unified listener of /${dsRoot},
*/ */
@ -59,14 +63,16 @@ public class ZookeeperCachedOperator extends ZookeeperOperator {
treeCache.start(); treeCache.start();
} catch (Exception e) { } catch (Exception e) {
logger.error("add listener to zk path: {} failed", getZookeeperConfig().getDsRoot()); logger.error("add listener to zk path: {} failed", getZookeeperConfig().getDsRoot());
throw new RuntimeException(e); throw new ServiceException(e);
} }
} }
//for sub class //for sub class
protected void dataChanged(final CuratorFramework client, final TreeCacheEvent event, final String path){} protected void dataChanged(final CuratorFramework client, final TreeCacheEvent event, final String path) {
// Used by sub class
}
public String getFromCache(final String cachePath, final String key) { public String getFromCache(final String key) {
ChildData resultInCache = treeCache.getCurrentData(key); ChildData resultInCache = treeCache.getCurrentData(key);
if (null != resultInCache) { if (null != resultInCache) {
return null == resultInCache.getData() ? null : new String(resultInCache.getData(), StandardCharsets.UTF_8); return null == resultInCache.getData() ? null : new String(resultInCache.getData(), StandardCharsets.UTF_8);
@ -74,11 +80,11 @@ public class ZookeeperCachedOperator extends ZookeeperOperator {
return null; return null;
} }
public TreeCache getTreeCache(final String cachePath) { public TreeCache getTreeCache() {
return treeCache; return treeCache;
} }
public void addListener(TreeCacheListener listener){ public void addListener(TreeCacheListener listener) {
this.treeCache.getListenable().addListener(listener); this.treeCache.getListenable().addListener(listener);
} }

45
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperOperator.java
Unescape View File

@ -14,13 +14,17 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.service.zk; package org.apache.dolphinscheduler.service.zk;
import org.apache.commons.lang.StringUtils; import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.service.exceptions.ServiceException;
import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.api.ACLProvider; import org.apache.curator.framework.api.ACLProvider;
import org.apache.curator.framework.api.transaction.CuratorOp;
import org.apache.curator.framework.state.ConnectionState; import org.apache.curator.framework.state.ConnectionState;
import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.curator.utils.CloseableUtils; import org.apache.curator.utils.CloseableUtils;
@ -29,18 +33,16 @@ import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Stat; import org.apache.zookeeper.data.Stat;
import java.nio.charset.StandardCharsets;
import java.util.List;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.TimeUnit;
import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull;
/** /**
* zk base operator * zk base operator
*/ */
@ -64,19 +66,23 @@ public class ZookeeperOperator implements InitializingBean {
/** /**
* this method is for sub class, * this method is for sub class,
*/ */
protected void registerListener(){} protected void registerListener() {
// Used by sub class
}
protected void treeCacheStart(){} protected void treeCacheStart() {
// Used by sub class
}
public void initStateLister() { public void initStateLister() {
checkNotNull(zkClient); checkNotNull(zkClient);
zkClient.getConnectionStateListenable().addListener((client, newState) -> { zkClient.getConnectionStateListenable().addListener((client, newState) -> {
if(newState == ConnectionState.LOST){ if (newState == ConnectionState.LOST) {
logger.error("connection lost from zookeeper"); logger.error("connection lost from zookeeper");
} else if(newState == ConnectionState.RECONNECTED){ } else if (newState == ConnectionState.RECONNECTED) {
logger.info("reconnected to zookeeper"); logger.info("reconnected to zookeeper");
} else if(newState == ConnectionState.SUSPENDED){ } else if (newState == ConnectionState.SUSPENDED) {
logger.warn("connection SUSPENDED to zookeeper"); logger.warn("connection SUSPENDED to zookeeper");
} }
}); });
@ -85,7 +91,8 @@ public class ZookeeperOperator implements InitializingBean {
private CuratorFramework buildClient() { private CuratorFramework buildClient() {
logger.info("zookeeper registry center init, server lists is: {}.", zookeeperConfig.getServerList()); logger.info("zookeeper registry center init, server lists is: {}.", zookeeperConfig.getServerList());
CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder().ensembleProvider(new DefaultEnsembleProvider(checkNotNull(zookeeperConfig.getServerList(),"zookeeper quorum can't be null"))) CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder().ensembleProvider(new DefaultEnsembleProvider(checkNotNull(zookeeperConfig.getServerList(),
"zookeeper quorum can't be null")))
.retryPolicy(new ExponentialBackoffRetry(zookeeperConfig.getBaseSleepTimeMs(), zookeeperConfig.getMaxRetries(), zookeeperConfig.getMaxSleepMs())); .retryPolicy(new ExponentialBackoffRetry(zookeeperConfig.getBaseSleepTimeMs(), zookeeperConfig.getMaxRetries(), zookeeperConfig.getMaxSleepMs()));
//these has default value //these has default value
@ -114,7 +121,7 @@ public class ZookeeperOperator implements InitializingBean {
try { try {
zkClient.blockUntilConnected(); zkClient.blockUntilConnected();
} catch (final Exception ex) { } catch (final Exception ex) {
throw new RuntimeException(ex); throw new ServiceException(ex);
} }
return zkClient; return zkClient;
} }
@ -138,12 +145,12 @@ public class ZookeeperOperator implements InitializingBean {
throw new IllegalStateException(ex); throw new IllegalStateException(ex);
} catch (Exception ex) { } catch (Exception ex) {
logger.error("getChildrenKeys key : {}", key, ex); logger.error("getChildrenKeys key : {}", key, ex);
throw new RuntimeException(ex); throw new ServiceException(ex);
} }
} }
public boolean hasChildren(final String key){ public boolean hasChildren(final String key) {
Stat stat ; Stat stat;
try { try {
stat = zkClient.checkExists().forPath(key); stat = zkClient.checkExists().forPath(key);
return stat.getNumChildren() >= 1; return stat.getNumChildren() >= 1;
@ -241,4 +248,4 @@ public class ZookeeperOperator implements InitializingBean {
public void close() { public void close() {
CloseableUtils.closeQuietly(zkClient); CloseableUtils.closeQuietly(zkClient);
} }
} }

Write Preview
Loading…
Cancel
Save