From fbf513c7a4ba442102ff8fc5b6cba009a5fea6bf Mon Sep 17 00:00:00 2001 From: baoliang Date: Fri, 10 May 2019 17:15:13 +0800 Subject: [PATCH 01/12] join parent parameters into sub process global parameters --- .../java/cn/escheduler/dao/ProcessDao.java | 48 ++++++++++++++----- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java index b18fb5e974..3be8ee4a83 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java @@ -33,6 +33,8 @@ import com.alibaba.fastjson.JSONObject; import com.cronutils.model.Cron; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hdfs.web.JsonUtil; +import org.apache.twill.internal.json.JsonUtils; import org.quartz.CronExpression; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -689,41 +691,63 @@ public class ProcessDao extends AbstractBaseDao { * handle sub work process instance, update relation table and command parameters * set sub work process flag, extends parent work process command parameters. */ - public ProcessInstance setSubProcessParam(ProcessInstance processInstance){ - String cmdParam = processInstance.getCommandParam(); + public ProcessInstance setSubProcessParam(ProcessInstance subProcessInstance){ + String cmdParam = subProcessInstance.getCommandParam(); if(StringUtils.isEmpty(cmdParam)){ - return processInstance; + return subProcessInstance; } Map paramMap = JSONUtils.toMap(cmdParam); // write sub process id into cmd param. if(paramMap.containsKey(CMDPARAM_SUB_PROCESS) && CMDPARAM_EMPTY_SUB_PROCESS.equals(paramMap.get(CMDPARAM_SUB_PROCESS))){ paramMap.remove(CMDPARAM_SUB_PROCESS); - paramMap.put(CMDPARAM_SUB_PROCESS, String.valueOf(processInstance.getId())); - processInstance.setCommandParam(JSONUtils.toJson(paramMap)); - processInstance.setIsSubProcess(Flag.YES); - this.saveProcessInstance(processInstance); + paramMap.put(CMDPARAM_SUB_PROCESS, String.valueOf(subProcessInstance.getId())); + subProcessInstance.setCommandParam(JSONUtils.toJson(paramMap)); + subProcessInstance.setIsSubProcess(Flag.YES); + this.saveProcessInstance(subProcessInstance); } // copy parent instance user def params to sub process.. String parentInstanceId = paramMap.get(CMDPARAM_SUB_PROCESS_PARENT_INSTANCE_ID); if(StringUtils.isNotEmpty(parentInstanceId)){ ProcessInstance parentInstance = findProcessInstanceDetailById(Integer.parseInt(parentInstanceId)); if(parentInstance != null){ - processInstance.setGlobalParams(parentInstance.getGlobalParams()); - this.saveProcessInstance(processInstance); + subProcessInstance.setGlobalParams( + joinGlobalParams(parentInstance.getGlobalParams(), subProcessInstance.getGlobalParams())); + this.saveProcessInstance(subProcessInstance); }else{ logger.error("sub process command params error, cannot find parent instance: {} ", cmdParam); } } ProcessInstanceMap processInstanceMap = JSONUtils.parseObject(cmdParam, ProcessInstanceMap.class); if(processInstanceMap == null || processInstanceMap.getParentProcessInstanceId() == 0){ - return processInstance; + return subProcessInstance; } // update sub process id to process map table - processInstanceMap.setProcessInstanceId(processInstance.getId()); + processInstanceMap.setProcessInstanceId(subProcessInstance.getId()); this.updateWorkProcessInstanceMap(processInstanceMap); - return processInstance; + return subProcessInstance; + } + + /** + * join parent global params into sub process. + * only the keys doesn't in sub process global would be joined. + * @param parentGlobalParams + * @param subGlobalParams + * @return + */ + private String joinGlobalParams(String parentGlobalParams, String subGlobalParams){ + Map parentMap = JSONUtils.toMap(parentGlobalParams); + Map subMap = JSONUtils.toMap(subGlobalParams); + + Set parentKeySet = parentMap.keySet(); + for(String parentKey : parentKeySet){ + if(subMap.containsKey(parentKey)){ + continue; + } + subMap.put( parentKey, parentMap.get(parentKey)); + } + return JSONUtils.toJson(subMap); } /** From 07e6095c77dc1a24b9bb1c97e1fc4e99ab1a71b1 Mon Sep 17 00:00:00 2001 From: baoliang Date: Fri, 10 May 2019 17:37:36 +0800 Subject: [PATCH 02/12] add esc --- .../main/java/cn/escheduler/dao/ProcessDao.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java index 3be8ee4a83..44262488fa 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java @@ -20,6 +20,7 @@ import cn.escheduler.common.Constants; import cn.escheduler.common.enums.*; import cn.escheduler.common.model.DateInterval; import cn.escheduler.common.model.TaskNode; +import cn.escheduler.common.process.Property; import cn.escheduler.common.queue.ITaskQueue; import cn.escheduler.common.queue.TaskQueueFactory; import cn.escheduler.common.task.subprocess.SubProcessParameters; @@ -43,6 +44,7 @@ import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; import java.util.*; +import java.util.stream.Collectors; import static cn.escheduler.common.Constants.*; import static cn.escheduler.dao.datasource.ConnectionFactory.getMapper; @@ -737,17 +739,16 @@ public class ProcessDao extends AbstractBaseDao { * @return */ private String joinGlobalParams(String parentGlobalParams, String subGlobalParams){ - Map parentMap = JSONUtils.toMap(parentGlobalParams); - Map subMap = JSONUtils.toMap(subGlobalParams); + List parentPropertyList = JSONUtils.toList(parentGlobalParams, Property.class); + List subPropertyList = JSONUtils.toList(subGlobalParams, Property.class); + Map subMap = subPropertyList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); - Set parentKeySet = parentMap.keySet(); - for(String parentKey : parentKeySet){ - if(subMap.containsKey(parentKey)){ - continue; + for(Property parent : parentPropertyList){ + if(!subMap.containsKey(parent.getProp())){ + subPropertyList.add(parent); } - subMap.put( parentKey, parentMap.get(parentKey)); } - return JSONUtils.toJson(subMap); + return JSONUtils.toJson(subPropertyList); } /** From ad18251c7a606a8551d3b1deb55070453e4ae8a3 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Thu, 16 May 2019 19:16:13 +0800 Subject: [PATCH 03/12] work tolerance update --- .../src/main/java/cn/escheduler/server/utils/AlertManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/utils/AlertManager.java b/escheduler-server/src/main/java/cn/escheduler/server/utils/AlertManager.java index fc62bcf73d..147f538bca 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/utils/AlertManager.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/utils/AlertManager.java @@ -175,7 +175,7 @@ public class AlertManager { alert.setContent(content); alert.setAlertType(AlertType.EMAIL); alert.setCreateTime(new Date()); - alert.setAlertGroupId(processInstance.getWarningGroupId()); + alert.setAlertGroupId(processInstance.getWarningGroupId() == null ? 1:processInstance.getWarningGroupId()); alert.setReceivers(processInstance.getProcessDefinition().getReceivers()); alert.setReceiversCc(processInstance.getProcessDefinition().getReceiversCc()); From a40fcc0efaebe84f86afafc2d33cb90a44e56de5 Mon Sep 17 00:00:00 2001 From: baoliang Date: Thu, 16 May 2019 19:30:17 +0800 Subject: [PATCH 04/12] fix bug: unit calculate error --- .../cn/escheduler/server/master/runner/MasterExecThread.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java index ae170b8925..f12726c3ab 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java @@ -869,7 +869,7 @@ public class MasterExecThread implements Runnable { } Date now = new Date(); - long runningTime = DateUtils.differMs(now, processInstance.getStartTime()); + long runningTime = DateUtils.diffMin(now, processInstance.getStartTime()); if(runningTime > processInstance.getTimeout()){ return true; From 0bf8e270a6b133d19402c2001724ee520b66c8d3 Mon Sep 17 00:00:00 2001 From: baoliang Date: Fri, 17 May 2019 11:14:19 +0800 Subject: [PATCH 05/12] fix bug: tolerance task need not to check retry times --- .../src/main/java/cn/escheduler/dao/ProcessDao.java | 6 +++++- .../src/main/java/cn/escheduler/dao/model/TaskInstance.java | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java index 44262488fa..54220f0ea9 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java @@ -923,7 +923,11 @@ public class ProcessDao extends AbstractBaseDao { taskInstance.setFlag(Flag.NO); updateTaskInstance(taskInstance); // crate new task instance - taskInstance.setRetryTimes(taskInstance.getRetryTimes() + 1 ); + if(taskInstance.getState() != ExecutionStatus.NEED_FAULT_TOLERANCE){ + taskInstance.setRetryTimes(taskInstance.getRetryTimes() + 1 ); + } + taskInstance.setEndTime(null); + taskInstance.setStartTime(new Date()); taskInstance.setFlag(Flag.YES); taskInstance.setHost(null); taskInstance.setId(0); diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/model/TaskInstance.java b/escheduler-dao/src/main/java/cn/escheduler/dao/model/TaskInstance.java index f37b1e4349..7f7981ef7c 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/model/TaskInstance.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/model/TaskInstance.java @@ -422,8 +422,12 @@ public class TaskInstance { if(this.isSubProcess()){ return false; } - return (this.getState().typeIsFailure() + if(this.getState() == ExecutionStatus.NEED_FAULT_TOLERANCE){ + return true; + }else { + return (this.getState().typeIsFailure() && this.getRetryTimes() < this.getMaxRetryTimes()); + } } public void setDependency(String dependency) { From 0d5b4abcbfe01d64ff22a70674a844cb67145562 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Fri, 17 May 2019 15:09:27 +0800 Subject: [PATCH 06/12] only one master restart need failure and tolerant task restart scheduler --- .../src/main/java/cn/escheduler/dao/ProcessDao.java | 8 ++++++++ .../main/java/cn/escheduler/server/zk/ZKMasterClient.java | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java index b18fb5e974..1c7fc969ae 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java @@ -1526,6 +1526,14 @@ public class ProcessDao extends AbstractBaseDao { } + public void selfFaultTolerant(int ... states){ + List processInstanceList = processInstanceMapper.listByStatus(states); + for (ProcessInstance processInstance:processInstanceList){ + selfFaultTolerant(processInstance); + } + + } + @Transactional(value = "TransactionManager",rollbackFor = Exception.class) public void selfFaultTolerant(ProcessInstance processInstance){ diff --git a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java index 624d0193be..fe3360484c 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java @@ -123,9 +123,9 @@ public class ZKMasterClient extends AbstractZKClient { // register master this.registMaster(); - // check if fault tolerance is required + // check if fault tolerance is required，failure and tolerance if (getActiveMasterNum() == 1) { - processDao.selfFaultTolerant(ExecutionStatus.RUNNING_EXEUTION.ordinal()); + processDao.selfFaultTolerant(ExecutionStatus.RUNNING_EXEUTION.ordinal(),ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal()); } } From d1490155c28df232500f0f7e3fe546abb538b0f5 Mon Sep 17 00:00:00 2001 From: huyuanming Date: Mon, 20 May 2019 10:09:05 +0800 Subject: [PATCH 07/12] env --- escheduler-ui/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-ui/.env b/escheduler-ui/.env index 5cd5ba736d..fc1b619f69 100644 --- a/escheduler-ui/.env +++ b/escheduler-ui/.env @@ -1,5 +1,5 @@ -# 后端接口地址 +# 后端接口地址11 API_BASE = http://192.168.xx.xx:12345 # 本地开发如需ip访问项目把"#"号去掉 From f420d24187ae3d9bdcaa633480c1630afccaa942 Mon Sep 17 00:00:00 2001 From: lilin Date: Mon, 29 Apr 2019 16:49:43 +0800 Subject: [PATCH 08/12] =?UTF-8?q?=E6=8F=90=E7=A4=BA=E4=BF=A1=E6=81=AF?= =?UTF-8?q?=E4=B8=8D=E6=AD=A3=E7=A1=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- escheduler-ui/src/js/module/i18n/locale/zh_CN.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js index e47aa84ec4..023e487df5 100644 --- a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -130,7 +130,7 @@ export default { 'Please enter database name': '请输入数据库名', 'jdbc connect parameters': 'jdbc连接参数', 'Test Connect': '测试连接', - 'Please enter resource name': '请输入资源名称', + 'Please enter resource name': '请输入数据源名称', 'Please enter IP/hostname': '请输入IP/主机名', 'jdbc connection parameters is not a correct JSON format': 'jdbc连接参数不是一个正确的JSON格式', '#': '编号', @@ -419,7 +419,7 @@ export default { 'Create token': '创建令牌', 'Edit token': '编辑令牌', 'Please enter the IP address separated by commas': '请输入IP地址多个用逗号隔开', - 'Note: Multiple IP addresses have been comma separated': '注意：多个IP地址已逗号分割', + 'Note: Multiple IP addresses have been comma separated': '注意：多个IP地址以逗号分割', 'Failure time': '失效时间', 'User': '用户', 'Please enter token': '请输入令牌', From 28ba72b1ebc7a4001ead6701e42a6dfaa572af68 Mon Sep 17 00:00:00 2001 From: lilin Date: Mon, 29 Apr 2019 16:49:08 +0800 Subject: [PATCH 09/12] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=A6=82=E6=9E=9C?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E6=B2=A1=E6=9C=89=E5=88=9B=E5=BB=BA=E8=B5=84?= =?UTF-8?q?=E6=BA=90=E6=97=B6=E7=BB=99=E7=94=A8=E6=88=B7=E6=8E=88=E6=9D=83?= =?UTF-8?q?=E8=B5=84=E6=BA=90=E4=BC=9A=E6=8A=A5=E7=A9=BA=E6=8C=87=E9=92=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/escheduler/api/service/ResourcesService.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java index e0ab90185d..291c59ea22 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java @@ -701,17 +701,19 @@ public class ResourcesService extends BaseService { if (checkAdmin(loginUser, result)) { return result; } - List resourceList = resourcesMapper.queryResourceExceptUserId(userId); - Set resourceSet = null; + List list ; if (resourceList != null && resourceList.size() > 0) { - resourceSet = new HashSet<>(resourceList); - + Set resourceSet = new HashSet<>(resourceList); List authedResourceList = resourcesMapper.queryAuthorizedResourceList(userId); getAuthorizedResourceList(resourceSet, authedResourceList); + list = new ArrayList<>(resourceSet); + }else { + list = new ArrayList<>(0); } - result.put(Constants.DATA_LIST, new ArrayList<>(resourceSet)); + + result.put(Constants.DATA_LIST, list); putMsg(result,Status.SUCCESS); return result; } From faf993e6b7bada68d9bbcea41c555c542ee1c4bb Mon Sep 17 00:00:00 2001 From: baoliang Date: Mon, 20 May 2019 14:44:52 +0800 Subject: [PATCH 10/12] delete unused code --- escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java index 670aa1485b..0eb65dbe14 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java @@ -34,8 +34,6 @@ import com.alibaba.fastjson.JSONObject; import com.cronutils.model.Cron; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.hdfs.web.JsonUtil; -import org.apache.twill.internal.json.JsonUtils; import org.quartz.CronExpression; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From e8c8487cb43fa7ddf0d8397a0f649ebddfd4b8aa Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Mon, 20 May 2019 15:01:57 +0800 Subject: [PATCH 11/12] solve nginx 504 Gateway Time-out --- escheduler-ui/install-escheduler-ui.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/escheduler-ui/install-escheduler-ui.sh b/escheduler-ui/install-escheduler-ui.sh index c6266b98b2..60bf2b5341 100755 --- a/escheduler-ui/install-escheduler-ui.sh +++ b/escheduler-ui/install-escheduler-ui.sh @@ -88,9 +88,9 @@ eschedulerConf(){ proxy_set_header remote_addr $E_remote_addr; proxy_set_header X-Forwarded-For $E_proxy_add_x_forwarded_for; proxy_http_version 1.1; - proxy_connect_timeout 4s; - proxy_read_timeout 30s; - proxy_send_timeout 12s; + proxy_connect_timeout 300s; + proxy_read_timeout 300s; + proxy_send_timeout 300s; proxy_set_header Upgrade $E_http_upgrade; proxy_set_header Connection "upgrade"; } @@ -184,7 +184,7 @@ if [[ $version -eq 6 ]]; then fi # centos 7 -if [[ $version -eq 7 ]]; then +if [[ $version -eq 7 ]]; then centos7 fi From 1e742eed2c680c8ae05c003722a4c341eaeefbb6 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Mon, 20 May 2019 15:16:15 +0800 Subject: [PATCH 12/12] .escheduler_env.sh update --- script/env/.escheduler_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/env/.escheduler_env.sh b/script/env/.escheduler_env.sh index 7a258cc75d..5a08343c84 100644 --- a/script/env/.escheduler_env.sh +++ b/script/env/.escheduler_env.sh @@ -6,4 +6,4 @@ export PYTHON_HOME=/opt/soft/python export JAVA_HOME=/opt/soft/java export HIVE_HOME=/opt/soft/hive -export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH \ No newline at end of file +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH \ No newline at end of file