From 19ccd4f5316d812c58e0a2b21c0fca546ad3ee1a Mon Sep 17 00:00:00 2001 From: bao liang <29528966+lenboo@users.noreply.github.com> Date: Tue, 15 Oct 2019 15:53:05 +0800 Subject: [PATCH] fix bug: get process definition list failed. #1026 (#1027) * update english documents * refactor zk client * update documents * update zkclient * update zkclient * update documents * add architecture-design * change i18n * update i18n * update english documents * add architecture-design * update english documents * update en-US documents * add architecture-design * update demo site * add mybatis plus model * modify mybatisplus * modify mybatisplus * change interface by mybatisplus * add unit test * refactor dao interface. * add unit test for dao... * add unit test for dao... * add unit test for dao... * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ScheduleMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProcessInstanceMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectUserMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/QueueMapper.xml # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProcessInstanceMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProjectUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/QueueMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ResourceUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ScheduleMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/SessionMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/TenantMapperTest.java * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ScheduleMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProcessInstanceMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectUserMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/QueueMapper.xml # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProcessInstanceMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProjectUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/QueueMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ResourceUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ScheduleMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/SessionMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/TenantMapperTest.java * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml * update some dao bugs * update for some bugs * update some bugs * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml * update * update * add multiply settings for application.yml * add multiply settings for application.yml * revert * update configuration settings in task record dao... * change application_master to application-master * change application_master to application-master * update application.yml to application.properties * revert * revert * add properties * add properties * revert * revert * add api start up.. add alert send try catch * update dao info level * fix bug: task cannot submit when recovery failover * fix bug: task cannot submit when recovery failover * merge from dev-db * revert * revert * fix bug: get process definition list failed. * fix bug: process instance interval is error * revert * revert --- .../api/service/ProcessDefinitionService.java | 8 ++++++-- .../api/service/ProcessInstanceService.java | 10 ++++------ .../org/apache/dolphinscheduler/common/Constants.java | 4 ++++ .../server/worker/task/dependent/DependentTask.java | 2 +- .../server/worker/task/http/HttpTask.java | 4 +++- .../server/worker/task/processdure/ProcedureTask.java | 3 ++- .../server/worker/task/sql/SqlTask.java | 3 ++- 7 files changed, 22 insertions(+), 12 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index 91cb385614..fdfd000414 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -893,11 +893,15 @@ public class ProcessDefinitionService extends BaseDAGService { public Map getTaskNodeListByDefinitionIdList(String defineIdList) throws Exception { Map result = new HashMap<>(); - Map> taskNodeMap = new HashMap<>(); String[] idList = defineIdList.split(","); List definitionIdList = Arrays.asList(idList); - List processDefinitionList = processDefineMapper.queryDefinitionListByIdList( definitionIdList.toArray(new Integer[definitionIdList.size()])); + List idIntList = new ArrayList<>(); + for(String definitionId : definitionIdList) { + idIntList.add(Integer.parseInt(definitionId)); + } + Integer[] idArray = idIntList.toArray(new Integer[idIntList.size()]); + List processDefinitionList = processDefineMapper.queryDefinitionListByIdList(idArray); if (processDefinitionList == null || processDefinitionList.size() ==0) { logger.info("process definition not exists"); putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, defineIdList); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java index 2239646787..34b80d9548 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java @@ -193,12 +193,10 @@ public class ProcessInstanceService extends BaseDAGService { processInstanceMapper.queryProcessInstanceListPaging(page, project.getId(), processDefineId, searchVal, statusArray, host, start, end); - for(ProcessInstance processInstance:processInstanceList.getRecords()){ - if (StringUtils.isNotEmpty(startDate) - && StringUtils.isNotEmpty(endDate)){ - processInstance.setDuration(DateUtils.differSec(processInstance.getStartTime(),processInstance.getEndTime())); - } + List processInstances = processInstanceList.getRecords(); + for(ProcessInstance processInstance: processInstances){ + processInstance.setDuration(DateUtils.differSec(processInstance.getStartTime(),processInstance.getEndTime())); } Set exclusionSet = new HashSet(){{ @@ -210,7 +208,7 @@ public class ProcessInstanceService extends BaseDAGService { PageInfo pageInfo = new PageInfo(pageNo, pageSize); pageInfo.setTotalCount((int) processInstanceList.getTotal()); - pageInfo.setLists(CollectionUtils.getListByExclusion(processInstanceList.getRecords(), exclusionSet)); + pageInfo.setLists(CollectionUtils.getListByExclusion(processInstances, exclusionSet)); result.put(Constants.DATA_LIST, pageInfo); putMsg(result, Status.SUCCESS); return result; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 02c1e1c47b..2240aa9b43 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -901,6 +901,10 @@ public final class Constants { */ public static final String LOGIN_USER_KEY_TAB_PATH = "login.user.keytab.path"; + /** + * task log info format + */ + public static final String TASK_LOG_INFO_FORMAT = "TaskLogInfo-%s"; /** * hive conf diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java index 8510265869..6fbef4acad 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java @@ -81,7 +81,7 @@ public class DependentTask extends AbstractTask { @Override public void handle(){ // set the name of the current thread - String threadLoggerInfoName = String.format("TaskLogInfo-%s", taskProps.getTaskAppId()); + String threadLoggerInfoName = String.format(Constants.TASK_LOG_INFO_FORMAT, taskProps.getTaskAppId()); Thread.currentThread().setName(threadLoggerInfoName); try{ diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java index 3a56abaf37..77467d7c71 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.worker.task.http; +import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.HttpMethod; import org.apache.dolphinscheduler.common.enums.HttpParametersType; import org.apache.dolphinscheduler.common.process.HttpProperty; @@ -53,6 +54,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; + /** * http task */ @@ -92,7 +94,7 @@ public class HttpTask extends AbstractTask { @Override public void handle() throws Exception { - String threadLoggerInfoName = String.format("TaskLogInfo-%s", taskProps.getTaskAppId()); + String threadLoggerInfoName = String.format(Constants.TASK_LOG_INFO_FORMAT, taskProps.getTaskAppId()); Thread.currentThread().setName(threadLoggerInfoName); long startTime = System.currentTimeMillis(); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java index 7a6aaac289..e31cb009ea 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.server.worker.task.processdure; +import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.Direct; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; @@ -82,7 +83,7 @@ public class ProcedureTask extends AbstractTask { @Override public void handle() throws Exception { // set the name of the current thread - String threadLoggerInfoName = String.format("TaskLogInfo-%s", taskProps.getTaskAppId()); + String threadLoggerInfoName = String.format(Constants.TASK_LOG_INFO_FORMAT, taskProps.getTaskAppId()); Thread.currentThread().setName(threadLoggerInfoName); logger.info("processdure type : {}, datasource : {}, method : {} , localParams : {}", diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index 73eb0d1489..604fd543d8 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -22,6 +22,7 @@ import com.alibaba.fastjson.serializer.SerializerFeature; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.EnumUtils; import org.apache.dolphinscheduler.alert.utils.MailUtils; +import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.common.enums.UdfType; @@ -103,7 +104,7 @@ public class SqlTask extends AbstractTask { @Override public void handle() throws Exception { // set the name of the current thread - String threadLoggerInfoName = String.format("TaskLogInfo-%s", taskProps.getTaskAppId()); + String threadLoggerInfoName = String.format(Constants.TASK_LOG_INFO_FORMAT, taskProps.getTaskAppId()); Thread.currentThread().setName(threadLoggerInfoName); logger.info(sqlParameters.toString()); logger.info("sql type : {}, datasource : {}, sql : {} , localParams : {},udfs : {},showType : {},connParams : {}",