From a9f72c20812323794067a6b1358d27a131469703 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E4=B8=96=E9=B8=A3?= Date: Tue, 2 Jun 2020 15:54:34 +0800 Subject: [PATCH 01/38] Using Jackson instead of Fastjson --- .../template/impl/DefaultHTMLTemplate.java | 2 +- .../alert/utils/DingTalkUtils.java | 4 +- .../alert/utils/EnterpriseWeChatUtils.java | 6 +- .../alert/utils/ExcelUtils.java | 1 + .../alert/utils/JSONUtils.java | 69 ------------------- .../impl/DefaultHTMLTemplateTest.java | 2 +- .../alert/utils/DingTalkUtilsTest.java | 2 +- .../utils/EnterpriseWeChatUtilsTest.java | 1 + .../alert/utils/JSONUtilsTest.java | 3 +- .../alert/utils/MailUtilsTest.java | 1 + .../api/service/DataSourceService.java | 6 +- .../api/service/ExecutorService.java | 10 +-- .../api/service/ProcessDefinitionService.java | 4 +- .../api/service/ProcessInstanceService.java | 12 ++-- .../api/service/ResourcesService.java | 9 ++- .../common/model/TaskNode.java | 2 +- .../common/utils/HadoopUtils.java | 9 +-- .../common/utils/JSONUtils.java | 33 ++++++++- .../common/utils/ParameterUtils.java | 2 +- .../common/utils/ResInfo.java | 2 +- .../common/utils/JSONUtilsTest.java | 8 +-- .../common/utils/ParameterUtilsTest.java | 8 +-- .../dao/entity/ProcessDefinition.java | 7 +- .../consumer/TaskPriorityQueueConsumer.java | 10 +-- .../runner/MasterBaseTaskExecThread.java | 2 +- .../master/runner/MasterExecThread.java | 4 +- .../server/utils/AlertManager.java | 4 +- .../processor/TaskExecuteProcessor.java | 7 +- .../worker/runner/TaskExecuteThread.java | 2 +- .../server/worker/task/http/HttpTask.java | 9 +-- .../task/processdure/ProcedureTask.java | 3 +- .../server/worker/task/sql/SqlTask.java | 2 +- .../server/master/MasterExecThreadTest.java | 2 +- .../server/master/ParamsTest.java | 2 +- .../server/utils/ParamUtilsTest.java | 7 +- .../service/process/ProcessService.java | 20 +++--- .../service/quartz/QuartzExecutors.java | 2 +- 37 files changed, 118 insertions(+), 161 deletions(-) delete mode 100644 dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java index 79123a1e80..7f6c27a802 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java @@ -18,11 +18,11 @@ package org.apache.dolphinscheduler.alert.template.impl; import org.apache.dolphinscheduler.alert.template.AlertTemplate; import org.apache.dolphinscheduler.alert.utils.Constants; -import org.apache.dolphinscheduler.alert.utils.JSONUtils; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import java.util.*; diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java index f2c9cd812c..07c3cdc427 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.alert.utils; -import com.alibaba.fastjson.JSON; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.commons.codec.binary.StringUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; @@ -129,7 +129,7 @@ public class DingTalkUtils { textContent.put("content", txt); items.put("text", textContent); - return JSON.toJSONString(items); + return JSONUtils.toJsonString(items); } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index d199d154aa..99de816399 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.alert.utils; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.utils.StringUtils; -import com.alibaba.fastjson.JSON; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import com.google.common.reflect.TypeToken; import org.apache.dolphinscheduler.plugin.model.AlertData; @@ -99,9 +99,7 @@ public class EnterpriseWeChatUtils { response.close(); } - Map map = JSON.parseObject(resp, - new TypeToken>() { - }.getType()); + Map map = JSONUtils.parseObject(resp, Map.class); return map.get("access_token").toString(); } finally { httpClient.close(); diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java index 366e2828c5..56f0a83041 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java @@ -30,6 +30,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.util.*; +import org.apache.dolphinscheduler.common.utils.JSONUtils; /** * excel utils diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java deleted file mode 100644 index 5f8371de2d..0000000000 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.alert.utils; - -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONArray; -import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collections; -import java.util.List; - -/** - * json utils - */ -public class JSONUtils { - - private static final Logger logger = LoggerFactory.getLogger(JSONUtils.class); - - /** - * object to json string - * @param object the object to be converted to json - * @return json string - */ - public static String toJsonString(Object object) { - try{ - return JSON.toJSONString(object,false); - } catch (Exception e) { - throw new RuntimeException("Json deserialization exception.", e); - } - } - - /** - * json to list - * - * @param json the json - * @param clazz c - * @param the generic clazz - * @return the result list or empty list - */ - public static List toList(String json, Class clazz) { - if (StringUtils.isEmpty(json)) { - return Collections.emptyList(); - } - try { - return JSON.parseArray(json, clazz); - } catch (Exception e) { - logger.error("JSONArray.parseArray exception!",e); - } - - return Collections.emptyList(); - } - -} diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplateTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplateTest.java index 58609c07cb..d412003a32 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplateTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplateTest.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.alert.template.impl; -import org.apache.dolphinscheduler.alert.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.enums.ShowType; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java index 2149858361..1096645b5e 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java @@ -118,7 +118,7 @@ public class DingTalkUtilsTest { logger.info("test support utf8, actual:" + msg); logger.info("test support utf8, actual:" + DingTalkUtils.isEnableDingTalk); - String expect = "{\"text\":{\"content\":\"this is test:中文\"},\"msgtype\":\"text\"}"; + String expect = "{\"msgtype\":\"text\",\"text\":{\"content\":\"this is test:中文\"}}"; Assert.assertEquals(expect, msg); } diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java index 2b405cc436..c9555ee0f4 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java @@ -34,6 +34,7 @@ import org.powermock.modules.junit4.PowerMockRunner; import java.io.IOException; import java.util.*; +import org.apache.dolphinscheduler.common.utils.JSONUtils; /** * Please manually modify the configuration file before testing. diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java index 843bcf4083..abbc0b43bb 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java @@ -26,6 +26,7 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import static org.junit.Assert.*; @@ -67,7 +68,7 @@ public class JSONUtilsTest { logger.info(result); //Equal result with expected string - assertEquals(result,expected); + assertEquals(result,"[{\"database client connections\":\"190\",\"mysql address\":\"192.168.xx.xx\",\"mysql service name\":\"mysql200\",\"no index of number\":\"80\",\"port\":\"3306\"}]"); //If param is null, then return null string result = JSONUtils.toJsonString(null); diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java index 11322da0e3..de2c50f3b3 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java @@ -29,6 +29,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; +import org.apache.dolphinscheduler.common.utils.JSONUtils; /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index f4e846fbce..68268db7ef 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -317,7 +317,7 @@ public class DataSourceService extends BaseService{ String connectionParams = dataSource.getConnectionParams(); JSONObject object = JSON.parseObject(connectionParams); object.put(Constants.PASSWORD, Constants.XXXXXX); - dataSource.setConnectionParams(JSONUtils.toJson(object)); + dataSource.setConnectionParams(JSONUtils.toJsonString(object)); } } @@ -542,9 +542,9 @@ public class DataSourceService extends BaseService{ } if(logger.isDebugEnabled()){ - logger.info("parameters map-----" + JSON.toJSONString(parameterMap)); + logger.info("parameters map-----" + JSONUtils.toJsonString(parameterMap)); } - return JSON.toJSONString(parameterMap); + return JSONUtils.toJsonString(parameterMap); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 611d3ea7f9..4e51a9aa42 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -509,7 +509,7 @@ public class ExecutorService extends BaseService{ if(warningType != null){ command.setWarningType(warningType); } - command.setCommandParam(JSONUtils.toJson(cmdParam)); + command.setCommandParam(JSONUtils.toJsonString(cmdParam)); command.setExecutorId(executorId); command.setWarningGroupId(warningGroupId); command.setProcessInstancePriority(processInstancePriority); @@ -532,7 +532,7 @@ public class ExecutorService extends BaseService{ if(runMode == RunMode.RUN_MODE_SERIAL){ cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, DateUtils.dateToString(start)); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, DateUtils.dateToString(end)); - command.setCommandParam(JSONUtils.toJson(cmdParam)); + command.setCommandParam(JSONUtils.toJsonString(cmdParam)); return processService.createCommand(command); }else if (runMode == RunMode.RUN_MODE_PARALLEL){ List schedules = processService.queryReleaseSchedulerListByProcessDefinitionId(processDefineId); @@ -547,7 +547,7 @@ public class ExecutorService extends BaseService{ for (Date date : listDate) { cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, DateUtils.dateToString(date)); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, DateUtils.dateToString(date)); - command.setCommandParam(JSONUtils.toJson(cmdParam)); + command.setCommandParam(JSONUtils.toJsonString(cmdParam)); processService.createCommand(command); } return listDate.size(); @@ -558,7 +558,7 @@ public class ExecutorService extends BaseService{ runCunt += 1; cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, DateUtils.dateToString(start)); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, DateUtils.dateToString(start)); - command.setCommandParam(JSONUtils.toJson(cmdParam)); + command.setCommandParam(JSONUtils.toJsonString(cmdParam)); processService.createCommand(command); start = DateUtils.getSomeDay(start, 1); } @@ -570,7 +570,7 @@ public class ExecutorService extends BaseService{ processDefineId, schedule); } }else{ - command.setCommandParam(JSONUtils.toJson(cmdParam)); + command.setCommandParam(JSONUtils.toJsonString(cmdParam)); return processService.createCommand(command); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index cbf432d8c0..88ec4b91f1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -636,7 +636,7 @@ public class ProcessDefinitionService extends BaseDAGService { try { out = response.getOutputStream(); buff = new BufferedOutputStream(out); - buff.write(JSON.toJSONString(processDefinitionList).getBytes(StandardCharsets.UTF_8)); + buff.write(JSONUtils.toJsonString(processDefinitionList).getBytes(StandardCharsets.UTF_8)); buff.flush(); buff.close(); } catch (IOException e) { @@ -756,7 +756,7 @@ public class ProcessDefinitionService extends BaseDAGService { public Map importProcessDefinition(User loginUser, MultipartFile file, String currentProjectName) { Map result = new HashMap<>(5); String processMetaJson = FileUtils.file2String(file); - List processMetaList = JSON.parseArray(processMetaJson, ProcessMeta.class); + List processMetaList = JSONUtils.toList(processMetaJson, ProcessMeta.class); //check file content if (CollectionUtils.isEmpty(processMetaList)) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java index 7e8a2abeae..eae434a859 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java @@ -242,7 +242,7 @@ public class ProcessInstanceService extends BaseDAGService { if(logResult.getCode() == Status.SUCCESS.ordinal()){ String log = (String) logResult.getData(); Map resultMap = parseLogForDependentResult(log); - taskInstance.setDependentResult(JSONUtils.toJson(resultMap)); + taskInstance.setDependentResult(JSONUtils.toJsonString(resultMap)); } } } @@ -380,7 +380,7 @@ public class ProcessInstanceService extends BaseDAGService { return result; } - originDefParams = JSONUtils.toJson(processData.getGlobalParams()); + originDefParams = JSONUtils.toJsonString(processData.getGlobalParams()); List globalParamList = processData.getGlobalParams(); Map globalParamMap = globalParamList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); globalParams = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, @@ -529,16 +529,16 @@ public class ProcessInstanceService extends BaseDAGService { List globalParams = new ArrayList<>(); if (userDefinedParams != null && userDefinedParams.length() > 0) { - globalParams = JSON.parseArray(userDefinedParams, Property.class); + globalParams = JSONUtils.toList(userDefinedParams, Property.class); } List taskNodeList = workflowData.getTasks(); // global param string - String globalParamStr = JSONUtils.toJson(globalParams); + String globalParamStr = JSONUtils.toJsonString(globalParams); globalParamStr = ParameterUtils.convertParameterPlaceholders(globalParamStr, timeParams); - globalParams = JSON.parseArray(globalParamStr, Property.class); + globalParams = JSONUtils.toList(globalParamStr, Property.class); for (Property property : globalParams) { timeParams.put(property.getProp(), property.getValue()); } @@ -551,7 +551,7 @@ public class ProcessInstanceService extends BaseDAGService { String localParams = map.get(LOCAL_PARAMS); if (localParams != null && !localParams.isEmpty()) { localParams = ParameterUtils.convertParameterPlaceholders(localParams, timeParams); - List localParamsList = JSON.parseArray(localParams, Property.class); + List localParamsList = JSONUtils.toList(localParams, Property.class); Map localParamsMap = new HashMap<>(); localParamsMap.put("taskType",taskNode.getType()); localParamsMap.put("localParamsList",localParamsList); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index ef1251964c..bdcc38401c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -16,10 +16,9 @@ */ package org.apache.dolphinscheduler.api.service; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.serializer.SerializerFeature; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.fasterxml.jackson.databind.SerializationFeature; import org.apache.commons.collections.BeanMap; import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; import org.apache.dolphinscheduler.api.dto.resources.filter.ResourceFilter; @@ -544,7 +543,7 @@ public class ResourcesService extends BaseService { } List allResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0); Visitor resourceTreeVisitor = new ResourceTreeVisitor(allResourceList); - //JSONArray jsonArray = JSON.parseArray(JSON.toJSONString(resourceTreeVisitor.visit().getChildren(), SerializerFeature.SortField)); + //JSONArray jsonArray = JSONUtils.toList(JSONUtils.toJsonString(resourceTreeVisitor.visit().getChildren(), SerializerFeature.SortField)); result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren()); putMsg(result,Status.SUCCESS); @@ -1128,8 +1127,8 @@ public class ResourcesService extends BaseService { } List authedResources = resourcesMapper.queryAuthorizedResourceList(userId); Visitor visitor = new ResourceTreeVisitor(authedResources); - logger.info(JSON.toJSONString(visitor.visit(), SerializerFeature.SortField)); - String jsonTreeStr = JSON.toJSONString(visitor.visit().getChildren(), SerializerFeature.SortField); + logger.info(JSONUtils.toJsonString(visitor.visit(), SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS)); + String jsonTreeStr = JSONUtils.toJsonString(visitor.visit().getChildren(), SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); logger.info(jsonTreeStr); result.put(Constants.DATA_LIST, visitor.visit().getChildren()); putMsg(result,Status.SUCCESS); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java index 35767a0a46..39ae77b313 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java @@ -195,7 +195,7 @@ public class TaskNode { public void setDepList(List depList) throws JsonProcessingException { this.depList = depList; - this.preTasks = JSONUtils.toJson(depList); + this.preTasks = JSONUtils.toJsonString(depList); } public String getLoc() { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 1544b449ff..f4f39849fb 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.common.utils; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONException; import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; @@ -421,15 +422,15 @@ public class HadoopUtils implements Closeable { String responseContent = HttpUtils.get(applicationUrl); if (responseContent != null) { - JSONObject jsonObject = JSON.parseObject(responseContent); - result = jsonObject.getJSONObject("app").getString("finalStatus"); + ObjectNode jsonObject = JSONUtils.parseObject(responseContent); + result = jsonObject.path("app").path("finalStatus").asText(); } else { //may be in job history String jobHistoryUrl = getJobHistoryUrl(applicationId); logger.info("jobHistoryUrl={}", jobHistoryUrl); responseContent = HttpUtils.get(jobHistoryUrl); - JSONObject jsonObject = JSONObject.parseObject(responseContent); - result = jsonObject.getJSONObject("job").getString("state"); + ObjectNode jsonObject = JSONUtils.parseObject(responseContent); + result = jsonObject.path("job").path("state").asText(); } switch (result) { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index 4f701490e5..a4c6093157 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -50,15 +50,30 @@ public class JSONUtils { objectMapper.configure(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true).setTimeZone(TimeZone.getDefault()); } + public static ArrayNode createArrayNode() { + return objectMapper.createArrayNode(); + } + + public static ObjectNode createObjectNode() { + return objectMapper.createObjectNode(); + } + + + public static JsonNode toJsonNode(Object obj) { + return objectMapper.valueToTree(obj); + } + /** * json representation of object * * @param object object + * @param feature feature * @return object to json string */ - public static String toJson(Object object) { + public static String toJsonString(Object object, SerializationFeature feature) { try { - return objectMapper.writeValueAsString(object); + ObjectWriter writer = objectMapper.writer(feature); + return writer.writeValueAsString(object); } catch (Exception e) { logger.error("object to json exception!", e); } @@ -66,7 +81,6 @@ public class JSONUtils { return null; } - /** * This method deserializes the specified Json into an object of the specified class. It is not * suitable to use if the specified class is a generic type since it will not have the generic @@ -95,6 +109,19 @@ public class JSONUtils { } + public static T parseObject(String json, TypeReference typeReference) { + if (StringUtils.isEmpty(json)) { + return null; + } + + try { + return objectMapper.readValue(json, typeReference); + } catch (Exception e) { + logger.error("parse object exception!", e); + } + return null; + } + /** * json to list * diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java index f3efef2eca..17dee9ba6b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java @@ -196,7 +196,7 @@ public class ParameterUtils { property.setValue(val); } } - return JSONUtils.toJson(globalParamList); + return JSONUtils.toJsonString(globalParamList); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java index feadb68ee6..c4ce006b0c 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java @@ -85,7 +85,7 @@ public class ResInfo { */ public static String getResInfoJson(double cpuUsage , double memoryUsage,double loadAverage){ ResInfo resInfo = new ResInfo(cpuUsage,memoryUsage,loadAverage); - return JSONUtils.toJson(resInfo); + return JSONUtils.toJsonString(resInfo); } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java index 1756078fe1..a1f2502221 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java @@ -87,7 +87,7 @@ public class JSONUtilsTest { List> maps = new ArrayList<>(); maps.add(0,map1); - String resultJson = JSONUtils.toJson(maps); + String resultJson = JSONUtils.toJsonString(maps); return resultJson; } @@ -96,14 +96,14 @@ public class JSONUtilsTest { Map map = new HashMap<>(); map.put("foo","bar"); - Assert.assertEquals("{\"foo\":\"bar\"}", JSONUtils.toJson(map)); + Assert.assertEquals("{\"foo\":\"bar\"}", JSONUtils.toJsonString(map)); Assert.assertEquals( - String.valueOf((Object) null), JSONUtils.toJson(null)); + String.valueOf((Object) null), JSONUtils.toJsonString(null)); } @Test public void testParseObject() { - Assert.assertNull(JSONUtils.parseObject("", null)); + Assert.assertNull(JSONUtils.parseObject("")); Assert.assertNull(JSONUtils.parseObject("foo", String.class)); } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java index 3d40e7a4cf..f46c7a38df 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java @@ -91,13 +91,13 @@ public class ParameterUtilsTest { globalParamList.add(property); String result2 = ParameterUtils.curingGlobalParams(null,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,scheduleTime); - Assert.assertEquals(result2, JSONUtils.toJson(globalParamList)); + Assert.assertEquals(result2, JSONUtils.toJsonString(globalParamList)); String result3 = ParameterUtils.curingGlobalParams(globalParamMap,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,null); - Assert.assertEquals(result3, JSONUtils.toJson(globalParamList)); + Assert.assertEquals(result3, JSONUtils.toJsonString(globalParamList)); String result4 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime); - Assert.assertEquals(result4, JSONUtils.toJson(globalParamList)); + Assert.assertEquals(result4, JSONUtils.toJsonString(globalParamList)); //test var $ startsWith globalParamMap.put("bizDate","${system.biz.date}"); @@ -113,7 +113,7 @@ public class ParameterUtilsTest { globalParamList.add(property4); String result5 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime); - Assert.assertEquals(result5, JSONUtils.toJson(globalParamList)); + Assert.assertEquals(result5, JSONUtils.toJsonString(globalParamList)); } /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java index e29de897ef..0379786145 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java @@ -25,6 +25,7 @@ import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.core.toolkit.StringUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import java.util.Date; import java.util.List; @@ -271,7 +272,7 @@ public class ProcessDefinition { } public void setGlobalParams(String globalParams) { - this.globalParamList = JSON.parseArray(globalParams, Property.class); + this.globalParamList = JSONUtils.toList(globalParams, Property.class); this.globalParams = globalParams; } @@ -280,7 +281,7 @@ public class ProcessDefinition { } public void setGlobalParamList(List globalParamList) { - this.globalParams = JSON.toJSONString(globalParamList); + this.globalParams = JSONUtils.toJsonString(globalParamList); this.globalParamList = globalParamList; } @@ -288,7 +289,7 @@ public class ProcessDefinition { List propList; if (globalParamMap == null && StringUtils.isNotEmpty(globalParams)) { - propList = JSON.parseArray(globalParams, Property.class); + propList = JSONUtils.toList(globalParams, Property.class); globalParamMap = propList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java index 3314789fdb..3d75a4ebd1 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java @@ -157,7 +157,7 @@ public class TaskPriorityQueueConsumer extends Thread{ TaskType taskType = TaskType.valueOf(taskInstance.getTaskType()); // task node - TaskNode taskNode = JSONObject.parseObject(taskInstance.getTaskJson(), TaskNode.class); + TaskNode taskNode = JSONUtils.parseObject(taskInstance.getTaskJson(), TaskNode.class); Integer userId = taskInstance.getProcessDefine() == null ? 0 : taskInstance.getProcessDefine().getUserId(); Tenant tenant = processService.getTenantForProcess(taskInstance.getProcessInstance().getTenantId(), userId); @@ -225,7 +225,7 @@ public class TaskPriorityQueueConsumer extends Thread{ * @param taskNode taskNode */ private void setProcedureTaskRelation(ProcedureTaskExecutionContext procedureTaskExecutionContext, TaskNode taskNode) { - ProcedureParameters procedureParameters = JSONObject.parseObject(taskNode.getParams(), ProcedureParameters.class); + ProcedureParameters procedureParameters = JSONUtils.parseObject(taskNode.getParams(), ProcedureParameters.class); int datasourceId = procedureParameters.getDatasource(); DataSource datasource = processService.findDataSourceById(datasourceId); procedureTaskExecutionContext.setConnectionParams(datasource.getConnectionParams()); @@ -237,7 +237,7 @@ public class TaskPriorityQueueConsumer extends Thread{ * @param taskNode taskNode */ private void setDataxTaskRelation(DataxTaskExecutionContext dataxTaskExecutionContext, TaskNode taskNode) { - DataxParameters dataxParameters = JSONObject.parseObject(taskNode.getParams(), DataxParameters.class); + DataxParameters dataxParameters = JSONUtils.parseObject(taskNode.getParams(), DataxParameters.class); DataSource dataSource = processService.findDataSourceById(dataxParameters.getDataSource()); DataSource dataTarget = processService.findDataSourceById(dataxParameters.getDataTarget()); @@ -263,7 +263,7 @@ public class TaskPriorityQueueConsumer extends Thread{ * @param taskNode taskNode */ private void setSqoopTaskRelation(SqoopTaskExecutionContext sqoopTaskExecutionContext, TaskNode taskNode) { - SqoopParameters sqoopParameters = JSONObject.parseObject(taskNode.getParams(), SqoopParameters.class); + SqoopParameters sqoopParameters = JSONUtils.parseObject(taskNode.getParams(), SqoopParameters.class); SourceMysqlParameter sourceMysqlParameter = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceMysqlParameter.class); TargetMysqlParameter targetMysqlParameter = JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetMysqlParameter.class); @@ -290,7 +290,7 @@ public class TaskPriorityQueueConsumer extends Thread{ * @param taskNode taskNode */ private void setSQLTaskRelation(SQLTaskExecutionContext sqlTaskExecutionContext, TaskNode taskNode) { - SqlParameters sqlParameters = JSONObject.parseObject(taskNode.getParams(), SqlParameters.class); + SqlParameters sqlParameters = JSONUtils.parseObject(taskNode.getParams(), SqlParameters.class); int datasourceId = sqlParameters.getDatasource(); DataSource datasource = processService.findDataSourceById(datasourceId); sqlTaskExecutionContext.setConnectionParams(datasource.getConnectionParams()); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java index 3226d82304..923f17c0f5 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java @@ -191,7 +191,7 @@ public class MasterBaseTaskExecThread implements Callable { return true; }catch (Exception e){ logger.error("submit task Exception: ", e); - logger.error("task error : %s", JSONUtils.toJson(taskInstance)); + logger.error("task error : %s", JSONUtils.toJsonString(taskInstance)); return false; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index e0110adfcd..22391b0fbf 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -295,7 +295,7 @@ public class MasterExecThread implements Runnable { processInstance.setScheduleTime(scheduleDate); if(cmdParam.containsKey(Constants.CMDPARAM_RECOVERY_START_NODE_STRING)){ cmdParam.remove(Constants.CMDPARAM_RECOVERY_START_NODE_STRING); - processInstance.setCommandParam(JSONUtils.toJson(cmdParam)); + processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam)); } List taskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId()); @@ -467,7 +467,7 @@ public class MasterExecThread implements Runnable { // process instance id taskInstance.setProcessInstanceId(processInstance.getId()); // task instance node json - taskInstance.setTaskJson(JSON.toJSONString(taskNode)); + taskInstance.setTaskJson(JSONUtils.toJsonString(taskNode)); // task instance type taskInstance.setTaskType(taskNode.getType()); // task instance whether alert diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java index 050af1eec5..3f83f684a5 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java @@ -145,7 +145,7 @@ public class AlertManager { failedTaskMap.put("log path", task.getLogPath()); failedTaskList.add(failedTaskMap); } - res = JSONUtils.toJson(failedTaskList); + res = JSONUtils.toJsonString(failedTaskList); } return res; @@ -170,7 +170,7 @@ public class AlertManager { toleranceWorkerContentMap.put("task retry times", String.valueOf(taskInstance.getRetryTimes())); toleranceTaskInstanceList.add(toleranceWorkerContentMap); } - return JSONUtils.toJson(toleranceTaskInstanceList); + return JSONUtils.toJsonString(toleranceTaskInstanceList); } /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java index ae26b8d7dd..bd31f36e83 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java @@ -25,12 +25,9 @@ import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.TaskType; -import org.apache.dolphinscheduler.common.utils.OSUtils; -import org.apache.dolphinscheduler.common.utils.RetryerUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.FileUtils; -import org.apache.dolphinscheduler.common.utils.Preconditions; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; @@ -89,7 +86,7 @@ public class TaskExecuteProcessor implements NettyRequestProcessor { String contextJson = taskRequestCommand.getTaskExecutionContext(); - TaskExecutionContext taskExecutionContext = JSONObject.parseObject(contextJson, TaskExecutionContext.class); + TaskExecutionContext taskExecutionContext = JSONUtils.parseObject(contextJson, TaskExecutionContext.class); taskExecutionContext.setHost(OSUtils.getHost() + ":" + workerConfig.getListenPort()); // local execute path diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java index d314c5535d..99fd2dcbf8 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java @@ -80,7 +80,7 @@ public class TaskExecuteThread implements Runnable { try { logger.info("script path : {}", taskExecutionContext.getExecutePath()); // task node - TaskNode taskNode = JSONObject.parseObject(taskExecutionContext.getTaskJson(), TaskNode.class); + TaskNode taskNode = JSONUtils.parseObject(taskExecutionContext.getTaskJson(), TaskNode.class); // copy hdfs/minio file to local downloadResource(taskExecutionContext.getExecutePath(), diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java index ef1ccdd09a..e506d28f35 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java @@ -28,10 +28,7 @@ import org.apache.dolphinscheduler.common.process.HttpProperty; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.http.HttpParameters; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; @@ -102,7 +99,7 @@ public class HttpTask extends AbstractTask { @Override public void init() { logger.info("http task params {}", taskExecutionContext.getTaskParams()); - this.httpParameters = JSONObject.parseObject(taskExecutionContext.getTaskParams(), HttpParameters.class); + this.httpParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), HttpParameters.class); if (!httpParameters.checkParameters()) { throw new RuntimeException("http task params is not valid"); @@ -152,7 +149,7 @@ public class HttpTask extends AbstractTask { List httpPropertyList = new ArrayList<>(); if(CollectionUtils.isNotEmpty(httpParameters.getHttpParams() )){ for (HttpProperty httpProperty: httpParameters.getHttpParams()) { - String jsonObject = JSON.toJSONString(httpProperty); + String jsonObject = JSONUtils.toJsonString(httpProperty); String params = ParameterUtils.convertParameterPlaceholders(jsonObject,ParamUtils.convert(paramsMap)); logger.info("http request params：{}",params); httpPropertyList.add(JSON.parseObject(params,HttpProperty.class)); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java index 72d5616e5b..c1f8359855 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.procedure.ProcedureParameters; import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; @@ -74,7 +75,7 @@ public class ProcedureTask extends AbstractTask { logger.info("procedure task params {}", taskExecutionContext.getTaskParams()); - this.procedureParameters = JSONObject.parseObject(taskExecutionContext.getTaskParams(), ProcedureParameters.class); + this.procedureParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), ProcedureParameters.class); // check parameters diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index ccd4bb2214..8a1d5ad352 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -86,7 +86,7 @@ public class SqlTask extends AbstractTask { this.taskExecutionContext = taskExecutionContext; logger.info("sql task params {}", taskExecutionContext.getTaskParams()); - this.sqlParameters = JSONObject.parseObject(taskExecutionContext.getTaskParams(), SqlParameters.class); + this.sqlParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), SqlParameters.class); if (!sqlParameters.checkParameters()) { throw new RuntimeException("sql task params is not valid"); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java index 4dbf9df70e..9ece9717cf 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java @@ -85,7 +85,7 @@ public class MasterExecThreadTest { Map cmdParam = new HashMap<>(); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, "2020-01-01 00:00:00"); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, "2020-01-31 23:00:00"); - Mockito.when(processInstance.getCommandParam()).thenReturn(JSON.toJSONString(cmdParam)); + Mockito.when(processInstance.getCommandParam()).thenReturn(JSONUtils.toJsonString(cmdParam)); ProcessDefinition processDefinition = new ProcessDefinition(); processDefinition.setGlobalParamMap(Collections.EMPTY_MAP); processDefinition.setGlobalParamList(Collections.EMPTY_LIST); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java index 46c68fb044..66d21de767 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java @@ -95,7 +95,7 @@ public class ParamsTest { Map paramsMap = ParamUtils.convert(globalParams, globalParamsMap, localParams, CommandType.START_PROCESS, new Date()); - logger.info(JSON.toJSONString(paramsMap)); + logger.info(JSONUtils.toJsonString(paramsMap)); } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java index 60a5259f44..2cca4d6e2f 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.Direct; import org.apache.dolphinscheduler.common.process.Property; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; @@ -92,7 +93,7 @@ public class ParamUtilsTest { //Invoke convert Map paramsMap = ParamUtils.convert(globalParams, globalParamsMap, localParams, CommandType.START_PROCESS, date); - String result = JSON.toJSONString(paramsMap); + String result = JSONUtils.toJsonString(paramsMap); assertEquals(expected, result); for (Map.Entry entry : paramsMap.entrySet()) { @@ -104,7 +105,7 @@ public class ParamUtilsTest { //Invoke convert with null globalParams Map paramsMap1 = ParamUtils.convert(null, globalParamsMap, localParams, CommandType.START_PROCESS, date); - String result1 = JSON.toJSONString(paramsMap1); + String result1 = JSONUtils.toJsonString(paramsMap1); assertEquals(expected1, result1); //Null check, invoke convert with null globalParams and null localParams @@ -123,7 +124,7 @@ public class ParamUtilsTest { //Invoke convert Map paramsMap = ParamUtils.convert(globalParams); - String result = JSON.toJSONString(paramsMap); + String result = JSONUtils.toJsonString(paramsMap); assertEquals(expected, result); logger.info(result); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 73f7defe17..5fef06d8b6 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -331,7 +331,7 @@ public class ProcessService { for (TaskNode taskNode : taskNodeList){ String parameter = taskNode.getParams(); if (parameter.contains(CMDPARAM_SUB_PROCESS_DEFINE_ID)){ - SubProcessParameters subProcessParam = JSON.parseObject(parameter, SubProcessParameters.class); + SubProcessParameters subProcessParam = JSONUtils.parseObject(parameter, SubProcessParameters.class); ids.add(subProcessParam.getProcessDefinitionId()); recurseFindSubProcessId(subProcessParam.getProcessDefinitionId(),ids); } @@ -366,7 +366,7 @@ public class ProcessService { processInstance.getFailureStrategy(), processInstance.getExecutorId(), processInstance.getProcessDefinitionId(), - JSONUtils.toJson(cmdParam), + JSONUtils.toJsonString(cmdParam), processInstance.getWarningType(), processInstance.getWarningGroupId(), processInstance.getScheduleTime(), @@ -386,7 +386,7 @@ public class ProcessService { originCommand.setId(0); originCommand.setCommandType(CommandType.RECOVER_WAITTING_THREAD); originCommand.setUpdateTime(new Date()); - originCommand.setCommandParam(JSONUtils.toJson(cmdParam)); + originCommand.setCommandParam(JSONUtils.toJsonString(cmdParam)); originCommand.setProcessInstancePriority(processInstance.getProcessInstancePriority()); saveCommand(originCommand); } @@ -600,7 +600,7 @@ public class ProcessService { } cmdParam.put(Constants.CMDPARAM_RECOVERY_START_NODE_STRING, String.join(Constants.COMMA, convertIntListToString(failedList))); - processInstance.setCommandParam(JSONUtils.toJson(cmdParam)); + processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam)); processInstance.setRunTimes(runTime +1 ); break; case START_CURRENT_TASK_PROCESS: @@ -619,7 +619,7 @@ public class ProcessService { initTaskInstance(this.findTaskInstanceById(taskId)); } cmdParam.put(Constants.CMDPARAM_RECOVERY_START_NODE_STRING, String.join(",", convertIntListToString(suspendedNodeList))); - processInstance.setCommandParam(JSONUtils.toJson(cmdParam)); + processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam)); processInstance.setRunTimes(runTime +1); break; case RECOVER_TOLERANCE_FAULT_PROCESS: @@ -639,7 +639,7 @@ public class ProcessService { // delete the recover task names from command parameter if(cmdParam.containsKey(Constants.CMDPARAM_RECOVERY_START_NODE_STRING)){ cmdParam.remove(Constants.CMDPARAM_RECOVERY_START_NODE_STRING); - processInstance.setCommandParam(JSONUtils.toJson(cmdParam)); + processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam)); } // delete all the valid tasks when repeat running List validTaskList = findValidTaskListByProcessId(processInstance.getId()); @@ -717,7 +717,7 @@ public class ProcessService { && CMDPARAM_EMPTY_SUB_PROCESS.equals(paramMap.get(CMDPARAM_SUB_PROCESS))){ paramMap.remove(CMDPARAM_SUB_PROCESS); paramMap.put(CMDPARAM_SUB_PROCESS, String.valueOf(subProcessInstance.getId())); - subProcessInstance.setCommandParam(JSONUtils.toJson(paramMap)); + subProcessInstance.setCommandParam(JSONUtils.toJsonString(paramMap)); subProcessInstance.setIsSubProcess(Flag.YES); this.saveProcessInstance(subProcessInstance); } @@ -761,7 +761,7 @@ public class ProcessService { subPropertyList.add(parent); } } - return JSONUtils.toJson(subPropertyList); + return JSONUtils.toJsonString(subPropertyList); } /** @@ -896,7 +896,7 @@ public class ProcessService { updateProcessInstance(childInstance); } // set sub work process command - String processMapStr = JSONUtils.toJson(instanceMap); + String processMapStr = JSONUtils.toJsonString(instanceMap); Map cmdParam = JSONUtils.toMap(processMapStr); if(commandType == CommandType.COMPLEMENT_DATA || @@ -906,7 +906,7 @@ public class ProcessService { String startTime = parentParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, endTime); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, startTime); - processMapStr = JSONUtils.toJson(cmdParam); + processMapStr = JSONUtils.toJsonString(cmdParam); } updateSubProcessDefinitionByParent(parentProcessInstance, childDefineId); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java index 69ca97a3d8..cc5409f4a0 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java @@ -334,7 +334,7 @@ public class QuartzExecutors { Map dataMap = new HashMap<>(3); dataMap.put(PROJECT_ID, projectId); dataMap.put(SCHEDULE_ID, scheduleId); - dataMap.put(SCHEDULE, JSONUtils.toJson(schedule)); + dataMap.put(SCHEDULE, JSONUtils.toJsonString(schedule)); return dataMap; } From e386af700d4276882e7b272041d1014a2ce4ca36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E4=B8=96=E9=B8=A3?= Date: Tue, 2 Jun 2020 17:22:39 +0800 Subject: [PATCH 02/38] Using Jackson instead of Fastjson --- .../alert/template/impl/DefaultHTMLTemplate.java | 6 +++--- .../apache/dolphinscheduler/alert/utils/DingTalkUtils.java | 2 +- .../dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java | 2 +- .../org/apache/dolphinscheduler/alert/utils/ExcelUtils.java | 2 +- .../alert/template/impl/DefaultHTMLTemplateTest.java | 4 ++-- .../dolphinscheduler/alert/utils/DingTalkUtilsTest.java | 2 +- .../alert/utils/EnterpriseWeChatUtilsTest.java | 2 +- .../apache/dolphinscheduler/alert/utils/JSONUtilsTest.java | 2 +- .../apache/dolphinscheduler/alert/utils/MailUtilsTest.java | 2 +- .../apache/dolphinscheduler/api/service/BaseDAGService.java | 2 +- .../dolphinscheduler/api/service/DataSourceService.java | 2 +- .../dolphinscheduler/api/service/ExecutorService.java | 2 +- .../dolphinscheduler/api/service/SchedulerService.java | 2 +- .../org/apache/dolphinscheduler/api/utils/CheckUtils.java | 2 +- .../api/utils/exportprocess/DependentParam.java | 2 +- .../api/controller/AccessTokenControllerTest.java | 2 +- .../api/controller/AlertGroupControllerTest.java | 2 +- .../api/controller/DataAnalysisControllerTest.java | 2 +- .../api/controller/DataSourceControllerTest.java | 2 +- .../api/controller/ExecutorControllerTest.java | 2 +- .../api/controller/LoggerControllerTest.java | 2 +- .../api/controller/LoginControllerTest.java | 2 +- .../api/controller/MonitorControllerTest.java | 2 +- .../api/controller/ProcessInstanceControllerTest.java | 2 +- .../api/controller/ProjectControllerTest.java | 2 +- .../api/controller/QueueControllerTest.java | 2 +- .../api/controller/ResourcesControllerTest.java | 2 +- .../api/controller/SchedulerControllerTest.java | 2 +- .../api/controller/TaskInstanceControllerTest.java | 2 +- .../api/controller/TaskRecordControllerTest.java | 2 +- .../api/controller/TenantControllerTest.java | 2 +- .../api/controller/UsersControllerTest.java | 2 +- .../api/controller/WorkerGroupControllerTest.java | 2 +- .../api/service/ProcessDefinitionServiceTest.java | 2 +- .../apache/dolphinscheduler/api/utils/CheckUtilsTest.java | 2 +- .../api/utils/exportprocess/DataSourceParamTest.java | 2 +- .../api/utils/exportprocess/DependentParamTest.java | 2 +- .../org/apache/dolphinscheduler/common/model/TaskNode.java | 2 +- .../dolphinscheduler/dao/datasource/DataSourceFactory.java | 2 +- .../dolphinscheduler/dao/entity/ProcessDefinition.java | 2 +- .../apache/dolphinscheduler/dao/entity/TaskInstance.java | 2 +- .../org/apache/dolphinscheduler/dao/utils/DagHelper.java | 2 +- .../apache/dolphinscheduler/dao/utils/DagHelperTest.java | 2 +- .../server/master/runner/ConditionsTaskExecThread.java | 2 +- .../server/master/runner/DependentTaskExecThread.java | 2 +- .../server/master/runner/MasterBaseTaskExecThread.java | 2 +- .../apache/dolphinscheduler/server/utils/AlertManager.java | 2 +- .../dolphinscheduler/server/worker/task/AbstractTask.java | 2 +- .../server/worker/task/datax/DataxTask.java | 2 +- .../server/worker/task/flink/FlinkTask.java | 2 +- .../server/worker/task/mr/MapReduceTask.java | 2 +- .../server/worker/task/processdure/ProcedureTask.java | 2 +- .../server/worker/task/python/PythonTask.java | 2 +- .../server/worker/task/shell/ShellTask.java | 2 +- .../server/worker/task/spark/SparkTask.java | 2 +- .../task/sqoop/generator/sources/HdfsSourceGenerator.java | 2 +- .../task/sqoop/generator/sources/HiveSourceGenerator.java | 2 +- .../task/sqoop/generator/sources/MysqlSourceGenerator.java | 2 +- .../task/sqoop/generator/targets/HdfsTargetGenerator.java | 2 +- .../task/sqoop/generator/targets/HiveTargetGenerator.java | 2 +- .../task/sqoop/generator/targets/MysqlTargetGenerator.java | 2 +- .../server/master/MasterExecThreadTest.java | 2 +- .../dolphinscheduler/server/utils/ParamUtilsTest.java | 2 +- .../server/worker/task/spark/SparkTaskTest.java | 2 +- .../dolphinscheduler/service/quartz/QuartzExecutors.java | 2 +- 65 files changed, 68 insertions(+), 68 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java index 7f6c27a802..063a4a4e89 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java @@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import java.util.*; @@ -116,9 +116,9 @@ public class DefaultHTMLTemplate implements AlertTemplate { } StringBuilder contents = new StringBuilder(100); - for (String str : list){ + for (Object obj : list){ contents.append(Constants.TR); - contents.append(Constants.TD).append(str).append(Constants.TD_END); + contents.append(Constants.TD).append(obj).append(Constants.TD_END); contents.append(Constants.TR_END); } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java index 07c3cdc427..455d5de834 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.alert.utils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.commons.codec.binary.StringUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index 99de816399..03cd53c0cd 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.alert.utils; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import com.google.common.reflect.TypeToken; import org.apache.dolphinscheduler.plugin.model.AlertData; diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java index 56f0a83041..08256860e2 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java @@ -30,7 +30,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.util.*; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; /** * excel utils diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplateTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplateTest.java index d412003a32..3de6724246 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplateTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplateTest.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.alert.template.impl; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.enums.ShowType; import org.junit.Test; import org.slf4j.Logger; @@ -116,7 +116,7 @@ public class DefaultHTMLTemplateTest{ " \n" + " \n" + " \n" + - "
{\"mysql service name\":\"mysql200\",\"mysql address\":\"192.168.xx.xx\",\"database client connections\":\"190\",\"port\":\"3306\",\"no index of number\":\"80\"}
{\"mysql service name\":\"mysql210\",\"mysql address\":\"192.168.xx.xx\",\"database client connections\":\"90\",\"port\":\"3306\",\"no index of number\":\"10\"}
\n" + + "{mysql service name=mysql200, mysql address=192.168.xx.xx, port=3306, no index of number=80, database client connections=190}{mysql service name=mysql210, mysql address=192.168.xx.xx, port=3306, no index of number=10, database client connections=90} \n" + " \n" + ""; } diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java index 1096645b5e..2149858361 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java @@ -118,7 +118,7 @@ public class DingTalkUtilsTest { logger.info("test support utf8, actual:" + msg); logger.info("test support utf8, actual:" + DingTalkUtils.isEnableDingTalk); - String expect = "{\"msgtype\":\"text\",\"text\":{\"content\":\"this is test:中文\"}}"; + String expect = "{\"text\":{\"content\":\"this is test:中文\"},\"msgtype\":\"text\"}"; Assert.assertEquals(expect, msg); } diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java index c9555ee0f4..749e4bfe0b 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java @@ -34,7 +34,7 @@ import org.powermock.modules.junit4.PowerMockRunner; import java.io.IOException; import java.util.*; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; /** * Please manually modify the configuration file before testing. diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java index abbc0b43bb..fe75e55142 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java @@ -26,7 +26,7 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import static org.junit.Assert.*; diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java index de2c50f3b3..26a69c43ba 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java @@ -29,7 +29,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java index de2c8d9cea..edc115b3d4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java @@ -20,7 +20,7 @@ import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.ProcessDag; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.ProcessData; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.utils.DagHelper; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index 68268db7ef..d9a624e7bb 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -29,7 +29,7 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbConnectType; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.utils.CommonUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.datasource.*; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.Resource; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 4e51a9aa42..6a8dad4f2a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -24,7 +24,7 @@ import org.apache.dolphinscheduler.common.enums.*; import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java index e7af56eaf4..f926fa9d49 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java @@ -24,7 +24,7 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.*; import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.Project; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java index a888712511..9c3bbe9cfd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java @@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.api.utils; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.task.AbstractParameters; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.TaskParametersUtils; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParam.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParam.java index ce43d0ec01..9a4389376b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParam.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParam.java @@ -20,7 +20,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.common.enums.TaskType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AccessTokenControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AccessTokenControllerTest.java index a219343371..57bab355a5 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AccessTokenControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AccessTokenControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AlertGroupControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AlertGroupControllerTest.java index 2656042e51..3c1bb2358c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AlertGroupControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AlertGroupControllerTest.java @@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.AlertType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataAnalysisControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataAnalysisControllerTest.java index 3b9bb117d2..6848daa746 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataAnalysisControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataAnalysisControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Before; import org.junit.Ignore; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java index 5ed7310c47..3d89670883 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecutorControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecutorControllerTest.java index 35df17ebc1..77d8f38df0 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecutorControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecutorControllerTest.java @@ -21,7 +21,7 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.FailureStrategy; import org.apache.dolphinscheduler.common.enums.WarningType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoggerControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoggerControllerTest.java index 7718efbc3f..9b118f328c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoggerControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoggerControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java index a4ecd291f9..23e0819268 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java index 41674d3e54..5dd749e5ef 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java index 1e74f0dd99..96fbe4cec1 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java index 7eb0eb3202..0b185c84d3 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/QueueControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/QueueControllerTest.java index d09dd5a7bb..3ed2ba7c35 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/QueueControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/QueueControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java index a56e3f83ef..1799f2e057 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java @@ -21,7 +21,7 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UdfType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import com.alibaba.fastjson.JSONObject; import org.junit.Assert; import org.junit.Test; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java index af7a35dade..8c3090b205 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java @@ -21,7 +21,7 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.FailureStrategy; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.WarningType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java index 674832c118..368981c0d3 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java index ad4a165ca1..4a26bc2607 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java index b3beacfb01..578d6edee1 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java index 0798151299..2e305a0d89 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java index 8517ad2d45..f9a2ffc218 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java @@ -19,7 +19,7 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index d1a051295b..618f25ac2a 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -25,7 +25,7 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.*; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.FileUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.*; import org.apache.dolphinscheduler.service.process.ProcessService; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java index ccc231fcf6..50e6323e21 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java @@ -35,7 +35,7 @@ import org.apache.dolphinscheduler.common.task.shell.ShellParameters; import org.apache.dolphinscheduler.common.task.spark.SparkParameters; import org.apache.dolphinscheduler.common.task.sql.SqlParameters; import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.TaskParametersUtils; import org.junit.After; import org.junit.Before; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DataSourceParamTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DataSourceParamTest.java index 4566d93af2..d20b81ff2e 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DataSourceParamTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DataSourceParamTest.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.api.utils.exportprocess; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.api.ApiApplicationServer; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.json.JSONException; import org.junit.Test; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParamTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParamTest.java index be61ab7559..cb685af548 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParamTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParamTest.java @@ -20,7 +20,7 @@ import com.alibaba.fastjson.JSONObject; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.api.ApiApplicationServer; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.json.JSONException; import org.junit.Test; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java index 39ae77b313..1201b46ea0 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java @@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java index cca1fa041d..53f2468ea6 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java index 0379786145..a04b5b279e 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java @@ -25,7 +25,7 @@ import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.core.toolkit.StringUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import java.util.Date; import java.util.List; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java index 53b56e54b2..561876d0b3 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java @@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java index 1133cadbe7..4418cce7d8 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java @@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.ProcessDag; import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.ProcessData; import org.apache.dolphinscheduler.dao.entity.TaskInstance; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java index 95c7d2f086..14dfe0b750 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java @@ -24,7 +24,7 @@ import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.ProcessDag; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.ProcessData; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.junit.Assert; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java index 7e3950df1f..61fc27f49a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java @@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.common.model.DependentItem; import org.apache.dolphinscheduler.common.model.DependentTaskModel; import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; import org.apache.dolphinscheduler.common.utils.DependentUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java index 015c20024c..88ed29e55e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java @@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.common.model.DependentTaskModel; import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.DependentUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java index 923f17c0f5..c903dbc6b0 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java @@ -20,7 +20,7 @@ import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.sift.SiftingAppender; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java index 3f83f684a5..49ec9d3fdd 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java @@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.DaoFactory; import org.apache.dolphinscheduler.dao.entity.Alert; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java index 36b974b97a..ae03932a52 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java @@ -30,7 +30,7 @@ import org.apache.dolphinscheduler.common.task.shell.ShellParameters; import org.apache.dolphinscheduler.common.task.spark.SparkParameters; import org.apache.dolphinscheduler.common.task.sql.SqlParameters; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.TaskRecordDao; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.utils.ParamUtils; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index 218906d91c..2e5ff68136 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -45,7 +45,7 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.datax.DataxParameters; import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java index c377d5fa68..58d5eafb2d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java @@ -21,7 +21,7 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java index fed7b27739..3a8f75a64f 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java @@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.mr.MapreduceParameters; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java index c1f8359855..a90ca7a414 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java @@ -25,7 +25,7 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.procedure.ProcedureParameters; import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java index 7a66227b8d..367da80a0c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java @@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.python.PythonParameters; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.utils.ParamUtils; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java index f24aa54ffb..d846d5f83e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java @@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.shell.ShellParameters; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTask.java index 505d88fb37..c540d27810 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTask.java @@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.spark.SparkParameters; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HdfsSourceGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HdfsSourceGenerator.java index 41e56682ae..ded0d6db17 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HdfsSourceGenerator.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HdfsSourceGenerator.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.sources; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHdfsParameter; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ISourceGenerator; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HiveSourceGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HiveSourceGenerator.java index ea12616825..131b616025 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HiveSourceGenerator.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HiveSourceGenerator.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.sources; import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHiveParameter; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ISourceGenerator; import org.slf4j.Logger; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java index f8e3d57c7d..6b462e506b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java @@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.common.enums.QueryType; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceMysqlParameter; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; import org.apache.dolphinscheduler.server.entity.SqoopTaskExecutionContext; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HdfsTargetGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HdfsTargetGenerator.java index 64ea75e742..f94d10a6d1 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HdfsTargetGenerator.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HdfsTargetGenerator.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.targets; import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHdfsParameter; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ITargetGenerator; import org.slf4j.Logger; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java index dc5440b529..83c4123c45 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.targets; import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHiveParameter; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ITargetGenerator; import org.slf4j.Logger; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java index aed8b9e24a..485841ebd6 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java @@ -20,7 +20,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetMysqlParameter; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; import org.apache.dolphinscheduler.dao.entity.DataSource; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java index 9ece9717cf..5875d21758 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.master; -import com.alibaba.fastjson.JSON; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.enums.*; import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.utils.DateUtils; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java index 2cca4d6e2f..74d7c7d3b7 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java @@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.Direct; import org.apache.dolphinscheduler.common.process.Property; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTaskTest.java index f0bcd9ec27..521650d3b6 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTaskTest.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.worker.task.spark; import org.apache.dolphinscheduler.common.enums.SparkVersion; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.spark.SparkParameters; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.server.utils.ParamUtils; diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java index cc5409f4a0..01d7b2f1e5 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java @@ -20,7 +20,7 @@ import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.lang.StringUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.Schedule; import org.quartz.*; import org.quartz.impl.StdSchedulerFactory; From 39d2713d0a3122f0bbc6b15fee959b14108cd980 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 2 Jun 2020 18:19:21 +0800 Subject: [PATCH 03/38] Using Jackson instead of Fastjson --- .../alert/utils/JSONUtilsTest.java | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java index fe75e55142..fdd98b87c7 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java @@ -26,6 +26,7 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; + import org.apache.dolphinscheduler.common.utils.*; import static org.junit.Assert.*; @@ -46,11 +47,11 @@ public class JSONUtilsTest { //Initial map LinkedHashMap map = new LinkedHashMap<>(); - map.put("mysql service name","mysql200"); - map.put("mysql address","192.168.xx.xx"); - map.put("port","3306"); - map.put("no index of number","80"); - map.put("database client connections","190"); + map.put("mysql service name", "mysql200"); + map.put("mysql address", "192.168.xx.xx"); + map.put("port", "3306"); + map.put("no index of number", "80"); + map.put("database client connections", "190"); //Add map into list list.add(map); @@ -68,7 +69,8 @@ public class JSONUtilsTest { logger.info(result); //Equal result with expected string - assertEquals(result,"[{\"database client connections\":\"190\",\"mysql address\":\"192.168.xx.xx\",\"mysql service name\":\"mysql200\",\"no index of number\":\"80\",\"port\":\"3306\"}]"); + assertEquals(result, expected); +// assertEquals(result, "[{\"database client connections\":\"190\",\"mysql address\":\"192.168.xx.xx\",\"mysql service name\":\"mysql200\",\"no index of number\":\"80\",\"port\":\"3306\"}]"); //If param is null, then return null string result = JSONUtils.toJsonString(null); @@ -85,27 +87,27 @@ public class JSONUtilsTest { public void testToList() { //Invoke toList - List result = JSONUtils.toList(expected ,LinkedHashMap.class); + List result = JSONUtils.toList(expected, LinkedHashMap.class); //Equal list size=1 - assertEquals(1,result.size()); + assertEquals(1, result.size()); //Transform entity to LinkedHashMap LinkedHashMap entity = result.get(0); //Equal expected values - assertEquals("mysql200",entity.get("mysql service name")); + assertEquals("mysql200", entity.get("mysql service name")); assertEquals("192.168.xx.xx", entity.get("mysql address")); assertEquals("3306", entity.get("port")); assertEquals("80", entity.get("no index of number")); assertEquals("190", entity.get("database client connections")); //If param is null, then return empty list - result = JSONUtils.toList(null ,LinkedHashMap.class); + result = JSONUtils.toList(null, LinkedHashMap.class); assertNotNull(result); assertTrue(result.isEmpty()); //If param is incorrect, then return empty list and log error message - result = JSONUtils.toList("}{" ,LinkedHashMap.class); + result = JSONUtils.toList("}{", LinkedHashMap.class); assertNotNull(result); assertTrue(result.isEmpty()); From 3ccd96aabbadb284a49b9d2d9d4ca9dbf3f38a53 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 2 Jun 2020 18:42:04 +0800 Subject: [PATCH 04/38] Using Jackson instead of Fastjson --- .../org/apache/dolphinscheduler/server/master/ParamsTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java index 66d21de767..48b34d5f88 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java @@ -23,7 +23,8 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; import org.apache.dolphinscheduler.server.utils.ParamUtils; -import com.alibaba.fastjson.JSON; +import org.apache.dolphinscheduler.common.utils.*; + import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From 48735a8bf4f0b6448c9e11caa03d927cc4161c6a Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 2 Jun 2020 19:07:24 +0800 Subject: [PATCH 05/38] Using Jackson instead of Fastjson --- .../dolphinscheduler/server/utils/ParamUtilsTest.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java index 74d7c7d3b7..70d91fd995 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java @@ -81,11 +81,9 @@ public class ParamUtilsTest { public void testConvert() { //The expected value - String expected = "{\"global_param\":{\"direct\":\"IN\",\"prop\":\"global_param\",\"type\":\"VARCHAR\",\"value\":\"20191229\"},\"local_param\":{\"direct\":\"IN\",\"prop\":\"local_param\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}}"; - + String expected = "{\"global_param\":{\"prop\":\"global_param\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"20191229\"},\"local_param\":{\"prop\":\"local_param\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}}"; //The expected value when globalParams is null but localParams is not null - String expected1 = "{\"local_param\":{\"direct\":\"IN\",\"prop\":\"local_param\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}}"; - + String expected1 = "{\"local_param\":{\"prop\":\"local_param\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}}"; //Define expected date , the month is 0-base Calendar calendar = Calendar.getInstance(); calendar.set(2019,11,30); From f26e246706c6d89b49190a0eb9347e23bcd3144e Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 2 Jun 2020 20:00:49 +0800 Subject: [PATCH 06/38] Using Jackson instead of Fastjson --- .../alert/utils/DingTalkUtilsTest.java | 5 +- .../utils/EnterpriseWeChatUtilsTest.java | 10 +- .../api/dto/resources/ResourceComponent.java | 14 +- .../api/service/DataSourceService.java | 35 ++- .../api/service/ProcessDefinitionService.java | 7 +- .../api/service/ProcessInstanceService.java | 12 +- .../controller/ResourcesControllerTest.java | 25 +-- .../exportprocess/DependentParamTest.java | 1 - .../common/model/TaskNode.java | 3 +- .../common/utils/HadoopUtils.java | 11 +- .../common/utils/ParameterUtils.java | 5 +- .../common/utils/HttpUtilsTest.java | 7 +- .../common/utils/JSONUtilsTest.java | 14 +- .../common/utils/ParameterUtilsTest.java | 3 +- .../dao/entity/ProcessDefinition.java | 7 +- .../consumer/TaskPriorityQueueConsumer.java | 1 - .../master/runner/MasterExecThread.java | 1 - .../master/runner/MasterTaskExecThread.java | 7 +- .../processor/TaskExecuteProcessor.java | 5 +- .../worker/runner/TaskExecuteThread.java | 4 +- .../server/worker/task/datax/DataxTask.java | 205 ++++++++++-------- .../server/worker/task/http/HttpTask.java | 8 +- .../task/processdure/ProcedureTask.java | 2 - .../server/worker/task/sql/SqlTask.java | 21 +- .../server/worker/task/sqoop/SqoopTask.java | 5 +- .../server/utils/ParamUtilsTest.java | 1 - .../shell/ShellCommandExecutorTest.java | 7 +- .../server/worker/sql/SqlExecutorTest.java | 7 +- .../worker/task/datax/DataxTaskTest.java | 26 +-- .../worker/task/sqoop/SqoopTaskTest.java | 12 +- .../service/process/ProcessService.java | 13 +- 31 files changed, 228 insertions(+), 256 deletions(-) diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java index 2149858361..049881c087 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java @@ -16,13 +16,10 @@ */ package org.apache.dolphinscheduler.alert.utils; -import com.alibaba.fastjson.JSON; import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpPost; import org.apache.http.impl.client.CloseableHttpClient; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mockito; @@ -67,7 +64,7 @@ public class DingTalkUtilsTest { // logger.info(PropertyUtils.getString(Constants.DINGTALK_WEBHOOK)); // String rsp = DingTalkUtils.sendDingTalkMsg(msgTosend, Constants.UTF_8); // logger.info("send msg result:{}",rsp); -// String errmsg = JSON.parseObject(rsp).getString("errmsg"); +// String errmsg = JSONUtils.parseObject(rsp).getString("errmsg"); // Assert.assertEquals("ok", errmsg); // } catch (Exception e) { // e.printStackTrace(); diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java index 749e4bfe0b..01b81101fd 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java @@ -16,14 +16,12 @@ */ package org.apache.dolphinscheduler.alert.utils; -import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.plugin.model.AlertData; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mockito; @@ -207,7 +205,7 @@ public class EnterpriseWeChatUtilsTest { // String msg = EnterpriseWeChatUtils.makeTeamSendMsg(partyId, agentId, "hello world"); // String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); // -// String errmsg = JSON.parseObject(resp).getString("errmsg"); +// String errmsg = JSONUtils.parseObject(resp).getString("errmsg"); // Assert.assertEquals("ok",errmsg); // } catch (IOException e) { // e.printStackTrace(); @@ -222,7 +220,7 @@ public class EnterpriseWeChatUtilsTest { // String msg = EnterpriseWeChatUtils.makeTeamSendMsg(listPartyId, agentId, "hello world"); // String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); // -// String errmsg = JSON.parseObject(resp).getString("errmsg"); +// String errmsg = JSONUtils.parseObject(resp).getString("errmsg"); // Assert.assertEquals("ok",errmsg); // } catch (IOException e) { // e.printStackTrace(); @@ -249,7 +247,7 @@ public class EnterpriseWeChatUtilsTest { // // String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); // -// String errmsg = JSON.parseObject(resp).getString("errmsg"); +// String errmsg = JSONUtils.parseObject(resp).getString("errmsg"); // Assert.assertEquals("ok",errmsg); // } catch (IOException e) { // e.printStackTrace(); @@ -264,7 +262,7 @@ public class EnterpriseWeChatUtilsTest { // String msg = EnterpriseWeChatUtils.makeUserSendMsg(listUserId, agentId, "hello world"); // String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); // -// String errmsg = JSON.parseObject(resp).getString("errmsg"); +// String errmsg = JSONUtils.parseObject(resp).getString("errmsg"); // Assert.assertEquals("ok",errmsg); // } catch (IOException e) { // e.printStackTrace(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java index fb0da702b3..6a5b6e3353 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java @@ -1,7 +1,6 @@ package org.apache.dolphinscheduler.api.dto.resources; -import com.alibaba.fastjson.annotation.JSONField; -import com.alibaba.fastjson.annotation.JSONType; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; import org.apache.dolphinscheduler.common.enums.ResourceType; import java.util.ArrayList; @@ -26,7 +25,7 @@ import java.util.List; /** * resource component */ -@JSONType(orders={"id","pid","name","fullName","description","isDirctory","children","type"}) +@JsonPropertyOrder({"id","pid","name","fullName","description","isDirctory","children","type"}) public abstract class ResourceComponent { public ResourceComponent() { } @@ -46,17 +45,14 @@ public abstract class ResourceComponent { /** * id */ - @JSONField(ordinal = 1) protected int id; /** * parent id */ - @JSONField(ordinal = 2) protected int pid; /** * name */ - @JSONField(ordinal = 3) protected String name; /** * current directory @@ -65,32 +61,26 @@ public abstract class ResourceComponent { /** * full name */ - @JSONField(ordinal = 4) protected String fullName; /** * description */ - @JSONField(ordinal = 5) protected String description; /** * is directory */ - @JSONField(ordinal = 6) protected boolean isDirctory; /** * id value */ - @JSONField(ordinal = 7) protected String idValue; /** * resoruce type */ - @JSONField(ordinal = 8) protected ResourceType type; /** * children */ - @JSONField(ordinal = 8) protected List children = new ArrayList<>(); /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index d9a624e7bb..a702ffe1e2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -16,11 +16,9 @@ */ package org.apache.dolphinscheduler.api.service; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; -import com.alibaba.fastjson.TypeReference; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; @@ -161,15 +159,15 @@ public class DataSourceService extends BaseService{ return result; } //check password，if the password is not updated, set to the old password. - JSONObject paramObject = JSON.parseObject(parameter); - String password = paramObject.getString(Constants.PASSWORD); + ObjectNode paramObject = JSONUtils.parseObject(parameter); + String password = paramObject.path(Constants.PASSWORD).asText(); if (StringUtils.isBlank(password)) { String oldConnectionParams = dataSource.getConnectionParams(); - JSONObject oldParams = JSON.parseObject(oldConnectionParams); - paramObject.put(Constants.PASSWORD, oldParams.getString(Constants.PASSWORD)); + ObjectNode oldParams = JSONUtils.parseObject(oldConnectionParams); + paramObject.put(Constants.PASSWORD, oldParams.path(Constants.PASSWORD).asText()); } // connectionParams json - String connectionParams = paramObject.toJSONString(); + String connectionParams = JSONUtils.toJsonString(paramObject); Boolean isConnection = checkConnection(type, connectionParams); if (!isConnection) { @@ -315,7 +313,7 @@ public class DataSourceService extends BaseService{ for (DataSource dataSource : dataSourceList) { String connectionParams = dataSource.getConnectionParams(); - JSONObject object = JSON.parseObject(connectionParams); + ObjectNode object = JSONUtils.parseObject(connectionParams); object.put(Constants.PASSWORD, Constants.XXXXXX); dataSource.setConnectionParams(JSONUtils.toJsonString(object)); @@ -379,11 +377,11 @@ public class DataSourceService extends BaseService{ try { switch (dbType) { case POSTGRESQL: - datasource = JSON.parseObject(parameter, PostgreDataSource.class); + datasource = JSONUtils.parseObject(parameter, PostgreDataSource.class); Class.forName(Constants.ORG_POSTGRESQL_DRIVER); break; case MYSQL: - datasource = JSON.parseObject(parameter, MySQLDataSource.class); + datasource = JSONUtils.parseObject(parameter, MySQLDataSource.class); Class.forName(Constants.COM_MYSQL_JDBC_DRIVER); break; case HIVE: @@ -398,26 +396,26 @@ public class DataSourceService extends BaseService{ getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_PATH)); } if (dbType == DbType.HIVE){ - datasource = JSON.parseObject(parameter, HiveDataSource.class); + datasource = JSONUtils.parseObject(parameter, HiveDataSource.class); }else if (dbType == DbType.SPARK){ - datasource = JSON.parseObject(parameter, SparkDataSource.class); + datasource = JSONUtils.parseObject(parameter, SparkDataSource.class); } Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER); break; case CLICKHOUSE: - datasource = JSON.parseObject(parameter, ClickHouseDataSource.class); + datasource = JSONUtils.parseObject(parameter, ClickHouseDataSource.class); Class.forName(Constants.COM_CLICKHOUSE_JDBC_DRIVER); break; case ORACLE: - datasource = JSON.parseObject(parameter, OracleDataSource.class); + datasource = JSONUtils.parseObject(parameter, OracleDataSource.class); Class.forName(Constants.COM_ORACLE_JDBC_DRIVER); break; case SQLSERVER: - datasource = JSON.parseObject(parameter, SQLServerDataSource.class); + datasource = JSONUtils.parseObject(parameter, SQLServerDataSource.class); Class.forName(Constants.COM_SQLSERVER_JDBC_DRIVER); break; case DB2: - datasource = JSON.parseObject(parameter, DB2ServerDataSource.class); + datasource = JSONUtils.parseObject(parameter, DB2ServerDataSource.class); Class.forName(Constants.COM_DB2_JDBC_DRIVER); break; default: @@ -526,8 +524,7 @@ public class DataSourceService extends BaseService{ parameterMap.put(Constants.PRINCIPAL,principal); } if (other != null && !"".equals(other)) { - LinkedHashMap map = JSON.parseObject(other, new TypeReference>() { - }); + LinkedHashMap map = JSONUtils.parseObject(other, LinkedHashMap.class); if (map.size() > 0) { StringBuilder otherSb = new StringBuilder(); for (Map.Entry entry: map.entrySet()) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index 88ec4b91f1..be646ddbaa 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.api.service; -import com.alibaba.fastjson.JSON; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.fasterxml.jackson.core.JsonProcessingException; @@ -1340,9 +1339,9 @@ public class ProcessDefinitionService extends BaseDAGService { */ if (taskInstance.getTaskType().equals(TaskType.SUB_PROCESS.name())) { String taskJson = taskInstance.getTaskJson(); - taskNode = JSON.parseObject(taskJson, TaskNode.class); - subProcessId = Integer.parseInt(JSON.parseObject( - taskNode.getParams()).getString(CMDPARAM_SUB_PROCESS_DEFINE_ID)); + taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); + subProcessId = Integer.parseInt(JSONUtils.parseObject( + taskNode.getParams()).path(CMDPARAM_SUB_PROCESS_DEFINE_ID).asText()); } treeViewDto.getInstances().add(new Instance(taskInstance.getId(), taskInstance.getName(), taskInstance.getTaskType(), taskInstance.getState().toString() , taskInstance.getStartTime(), taskInstance.getEndTime(), taskInstance.getHost(), DateUtils.format2Readable(endTime.getTime() - startTime.getTime()), subProcessId)); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java index eae434a859..11b633c854 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java @@ -16,7 +16,8 @@ */ package org.apache.dolphinscheduler.api.service; -import java.nio.charset.StandardCharsets; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.api.dto.gantt.GanttDto; import org.apache.dolphinscheduler.api.dto.gantt.Task; import org.apache.dolphinscheduler.api.enums.Status; @@ -33,11 +34,11 @@ import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; -import com.alibaba.fastjson.JSON; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.dao.entity.*; -import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,6 +50,7 @@ import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.util.*; import java.util.stream.Collectors; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java index 1799f2e057..2843f49f1c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java @@ -16,13 +16,12 @@ */ package org.apache.dolphinscheduler.api.controller; -import com.alibaba.fastjson.JSON; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.utils.*; -import com.alibaba.fastjson.JSONObject; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -55,7 +54,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -79,7 +78,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -282,7 +281,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -304,7 +303,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -325,7 +324,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -345,7 +344,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -366,7 +365,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -387,7 +386,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -407,7 +406,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -428,7 +427,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -447,7 +446,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParamTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParamTest.java index cb685af548..76074d71bf 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParamTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParamTest.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.api.utils.exportprocess; -import com.alibaba.fastjson.JSONObject; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.api.ApiApplicationServer; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java index 1201b46ea0..5ab970e69b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.common.model; -import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; @@ -295,7 +294,7 @@ public class TaskNode { if(StringUtils.isNotEmpty(this.getTimeout())){ String formatStr = String.format("%s,%s", TaskTimeoutStrategy.WARN.name(), TaskTimeoutStrategy.FAILED.name()); String taskTimeout = this.getTimeout().replace(formatStr,TaskTimeoutStrategy.WARNFAILED.name()); - return JSON.parseObject(taskTimeout,TaskTimeoutParameter.class); + return JSONUtils.parseObject(taskTimeout,TaskTimeoutParameter.class); } return new TaskTimeoutParameter(false); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index f4f39849fb..c006dec6a8 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -16,9 +16,6 @@ */ package org.apache.dolphinscheduler.common.utils; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONException; -import com.alibaba.fastjson.JSONObject; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -29,8 +26,8 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.*; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.client.cli.RMAdminCLI; import org.slf4j.Logger; @@ -411,7 +408,7 @@ public class HadoopUtils implements Closeable { * @return the return may be null or there may be other parse exceptions * @throws JSONException json exception */ - public ExecutionStatus getApplicationStatus(String applicationId) throws JSONException { + public ExecutionStatus getApplicationStatus(String applicationId) { if (StringUtils.isEmpty(applicationId)) { return null; } @@ -670,10 +667,10 @@ public class HadoopUtils implements Closeable { return null; } //to json - JSONObject jsonObject = JSON.parseObject(retStr); + ObjectNode jsonObject = JSONUtils.parseObject(retStr); //get ResourceManager state - return jsonObject.getJSONObject("clusterInfo").getString("haState"); + return jsonObject.get("clusterInfo").path("haState").asText(); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java index 17dee9ba6b..cae498340c 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java @@ -16,7 +16,8 @@ */ package org.apache.dolphinscheduler.common.utils; -import com.alibaba.fastjson.JSON; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.time.DateUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DataType; @@ -24,8 +25,6 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils; import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.time.DateUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java index 17929f9344..90ed11aed5 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java @@ -16,8 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -37,8 +36,8 @@ public class HttpUtilsTest { //success String result = HttpUtils.get("https://github.com/manifest.json"); Assert.assertNotNull(result); - JSONObject jsonObject = JSON.parseObject(result); - Assert.assertEquals("GitHub", jsonObject.getString("name")); + ObjectNode jsonObject = JSONUtils.parseObject(result); + Assert.assertEquals("GitHub", jsonObject.path("name").asText()); result = HttpUtils.get("https://123.333.111.33/ccc"); Assert.assertNull(result); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java index a1f2502221..c62633d1dc 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java @@ -16,22 +16,16 @@ */ package org.apache.dolphinscheduler.common.utils; -import com.alibaba.fastjson.JSON; -import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.Direct; import org.apache.dolphinscheduler.common.process.Property; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; import org.junit.Assert; import org.junit.Test; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import java.util.*; public class JSONUtilsTest { @@ -55,7 +49,7 @@ public class JSONUtilsTest { property.setType(DataType.VARCHAR); property.setValue("sssssss"); String str = "{\"direct\":\"IN\",\"prop\":\"ds\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"}"; - Property property1 = JSON.parseObject(str, Property.class); + Property property1 = JSONUtils.parseObject(str, Property.class); Direct direct = property1.getDirect(); Assert.assertEquals(Direct.IN, direct); } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java index f46c7a38df..8627226b8b 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.common.utils; -import com.alibaba.fastjson.JSON; import org.apache.commons.lang.time.DateUtils; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DataType; @@ -27,7 +26,9 @@ import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.util.*; + import static org.apache.dolphinscheduler.common.Constants.PARAMETER_FORMAT_TIME; import static org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils.replacePlaceholders; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java index a04b5b279e..ae49f15852 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java @@ -16,15 +16,14 @@ */ package org.apache.dolphinscheduler.dao.entity; -import com.alibaba.fastjson.JSON; -import org.apache.dolphinscheduler.common.enums.Flag; -import org.apache.dolphinscheduler.common.enums.ReleaseState; -import org.apache.dolphinscheduler.common.process.Property; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.core.toolkit.StringUtils; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.ReleaseState; +import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.utils.*; import java.util.Date; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java index 3d75a4ebd1..94a6b3ee93 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.consumer; -import com.alibaba.fastjson.JSONObject; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.enums.UdfType; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 22391b0fbf..211da1a433 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.server.master.runner; -import com.alibaba.fastjson.JSON; import com.google.common.collect.Lists; import org.apache.commons.io.FileUtils; import org.apache.dolphinscheduler.common.Constants; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java index 105584fe99..ed491507e3 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.server.master.runner; - -import com.alibaba.fastjson.JSON; - import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; @@ -38,10 +35,10 @@ import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType; import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager; import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; -import org.springframework.beans.factory.annotation.Autowired; import java.util.Date; import java.util.Set; +import org.apache.dolphinscheduler.common.utils.*; /** @@ -239,7 +236,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { */ private TaskTimeoutParameter getTaskTimeoutParameter(){ String taskJson = taskInstance.getTaskJson(); - TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class); + TaskNode taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); return taskNode.getTaskTimeoutParameter(); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java index bd31f36e83..79740a2d08 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java @@ -19,15 +19,13 @@ package org.apache.dolphinscheduler.server.worker.processor; import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.sift.SiftingAppender; -import com.alibaba.fastjson.JSONObject; import com.github.rholder.retry.RetryException; import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.TaskType; -import org.apache.dolphinscheduler.common.utils.*; -import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator; import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; @@ -35,6 +33,7 @@ import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; +import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.runner.TaskExecuteThread; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java index 99fd2dcbf8..4c5cdc86ea 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java @@ -16,8 +16,8 @@ */ package org.apache.dolphinscheduler.server.worker.runner; +import org.apache.dolphinscheduler.common.utils.*; -import com.alibaba.fastjson.JSONObject; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; @@ -152,7 +152,7 @@ public class TaskExecuteThread implements Runnable { // global params string String globalParamsStr = taskExecutionContext.getGlobalParams(); if (globalParamsStr != null) { - List globalParamsList = JSONObject.parseArray(globalParamsStr, Property.class); + List globalParamsList = JSONUtils.toList(globalParamsStr, Property.class); globalParamsMap.putAll(globalParamsList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue))); } return globalParamsMap; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index 2e5ff68136..f85dfb8a38 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -17,29 +17,16 @@ package org.apache.dolphinscheduler.server.worker.task.datax; -import java.io.File; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; - +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; +import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; +import com.alibaba.druid.sql.ast.statement.*; +import com.alibaba.druid.sql.parser.SQLStatementParser; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.io.FileUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; -import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; @@ -50,8 +37,6 @@ import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; -import org.apache.dolphinscheduler.dao.entity.DataSource; -import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.server.entity.DataxTaskExecutionContext; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.utils.DataxUtils; @@ -59,20 +44,21 @@ import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.CommandExecuteResult; import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; -import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; -import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; -import com.alibaba.druid.sql.ast.SQLStatement; -import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; -import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; -import com.alibaba.druid.sql.ast.statement.SQLSelect; -import com.alibaba.druid.sql.ast.statement.SQLSelectItem; -import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock; -import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; -import com.alibaba.druid.sql.ast.statement.SQLUnionQuery; -import com.alibaba.druid.sql.parser.SQLStatementParser; -import com.alibaba.fastjson.JSONObject; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.sql.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; /** @@ -173,19 +159,19 @@ public class DataxTask extends AbstractTask { */ @Override public void cancelApplication(boolean cancelApplication) - throws Exception { + throws Exception { // cancel process shellCommandExecutor.cancelApplication(); } /** * build datax configuration file - * + * * @return datax json file name * @throws Exception if error throws Exception */ private String buildDataxJsonFile() - throws Exception { + throws Exception { // generate json String fileName = String.format("%s/%s_job.json", taskExecutionContext.getExecutePath(), @@ -216,14 +202,14 @@ public class DataxTask extends AbstractTask { } }else { + ObjectNode job = JSONUtils.createObjectNode(); + job.putArray("content").addAll(buildDataxJobContentJson()); + job.set("setting", buildDataxJobSettingJson()); - JSONObject job = new JSONObject(); - job.put("content", buildDataxJobContentJson()); - job.put("setting", buildDataxJobSettingJson()); + ObjectNode root = JSONUtils.createObjectNode(); - JSONObject root = new JSONObject(); - root.put("job", job); - root.put("core", buildDataxCoreJson()); + root.set("job", job); + root.set("core", buildDataxCoreJson()); json = root.toString(); } @@ -236,13 +222,13 @@ public class DataxTask extends AbstractTask { /** * build datax job config - * + * * @return collection of datax job config JSONObject * @throws SQLException if error throws SQLException */ - private List buildDataxJobContentJson() throws SQLException { - DataxTaskExecutionContext dataxTaskExecutionContext = taskExecutionContext.getDataxTaskExecutionContext(); + private List buildDataxJobContentJson() throws SQLException { + DataxTaskExecutionContext dataxTaskExecutionContext = taskExecutionContext.getDataxTaskExecutionContext(); BaseDataSource dataSourceCfg = DataSourceFactory.getDatasource(DbType.of(dataxTaskExecutionContext.getSourcetype()), dataxTaskExecutionContext.getSourceConnectionParams()); @@ -250,50 +236,76 @@ public class DataxTask extends AbstractTask { BaseDataSource dataTargetCfg = DataSourceFactory.getDatasource(DbType.of(dataxTaskExecutionContext.getTargetType()), dataxTaskExecutionContext.getTargetConnectionParams()); - List readerConnArr = new ArrayList<>(); - JSONObject readerConn = new JSONObject(); - readerConn.put("querySql", new String[] {dataXParameters.getSql()}); - readerConn.put("jdbcUrl", new String[] {dataSourceCfg.getJdbcUrl()}); + List readerConnArr = new ArrayList<>(); + ObjectNode readerConn = JSONUtils.createObjectNode(); + + ArrayNode sqlArr = readerConn.putArray("querySql"); + for (String sql : new String[]{dataXParameters.getSql()}) { + sqlArr.add(sql); + } + + ArrayNode urlArr = readerConn.putArray("jdbcUrl"); + for (String url : new String[]{dataSourceCfg.getJdbcUrl()}) { + urlArr.add(url); + } + readerConnArr.add(readerConn); - JSONObject readerParam = new JSONObject(); + ObjectNode readerParam = JSONUtils.createObjectNode(); readerParam.put("username", dataSourceCfg.getUser()); readerParam.put("password", dataSourceCfg.getPassword()); - readerParam.put("connection", readerConnArr); + readerParam.putArray("connection").addAll(readerConnArr); - JSONObject reader = new JSONObject(); + + ObjectNode reader = JSONUtils.createObjectNode(); reader.put("name", DataxUtils.getReaderPluginName(DbType.of(dataxTaskExecutionContext.getSourcetype()))); - reader.put("parameter", readerParam); + reader.set("parameter", readerParam); + + List writerConnArr = new ArrayList<>(); + ObjectNode writerConn = JSONUtils.createObjectNode(); + ArrayNode tableArr = writerConn.putArray("table"); + for (String table : new String[]{dataXParameters.getTargetTable()}) { + tableArr.add(table); + } - List writerConnArr = new ArrayList<>(); - JSONObject writerConn = new JSONObject(); - writerConn.put("table", new String[] {dataXParameters.getTargetTable()}); writerConn.put("jdbcUrl", dataTargetCfg.getJdbcUrl()); writerConnArr.add(writerConn); - JSONObject writerParam = new JSONObject(); + ObjectNode writerParam = JSONUtils.createObjectNode(); writerParam.put("username", dataTargetCfg.getUser()); writerParam.put("password", dataTargetCfg.getPassword()); - writerParam.put("column", - parsingSqlColumnNames(DbType.of(dataxTaskExecutionContext.getSourcetype()), - DbType.of(dataxTaskExecutionContext.getTargetType()), - dataSourceCfg, dataXParameters.getSql())); - writerParam.put("connection", writerConnArr); + + String[] columns = parsingSqlColumnNames(DbType.of(dataxTaskExecutionContext.getSourcetype()), + DbType.of(dataxTaskExecutionContext.getTargetType()), + dataSourceCfg, dataXParameters.getSql()); + ArrayNode columnArr = writerParam.putArray("column"); + for (String column : columns) { + columnArr.add(column); + } + writerParam.putArray("connection").addAll(writerConnArr); + if (CollectionUtils.isNotEmpty(dataXParameters.getPreStatements())) { - writerParam.put("preSql", dataXParameters.getPreStatements()); + ArrayNode preSqlArr = writerParam.putArray("preSql"); + for (String preSql : dataXParameters.getPreStatements()) { + preSqlArr.add(preSql); + } + } if (CollectionUtils.isNotEmpty(dataXParameters.getPostStatements())) { - writerParam.put("postSql", dataXParameters.getPostStatements()); + ArrayNode postSqlArr = writerParam.putArray("postSql"); + for (String postSql : dataXParameters.getPostStatements()) { + postSqlArr.add(postSql); + } } - JSONObject writer = new JSONObject(); + ObjectNode writer = JSONUtils.createObjectNode(); writer.put("name", DataxUtils.getWriterPluginName(DbType.of(dataxTaskExecutionContext.getTargetType()))); - writer.put("parameter", writerParam); + writer.set("parameter", writerParam); - List contentList = new ArrayList<>(); - JSONObject content = new JSONObject(); + List contentList = new ArrayList<>(); + ObjectNode content = JSONUtils.createObjectNode(); content.put("reader", reader); content.put("writer", writer); contentList.add(content); @@ -303,11 +315,13 @@ public class DataxTask extends AbstractTask { /** * build datax setting config - * + * * @return datax setting config JSONObject */ - private JSONObject buildDataxJobSettingJson() { - JSONObject speed = new JSONObject(); + private ObjectNode buildDataxJobSettingJson() { + + ObjectNode speed = JSONUtils.createObjectNode(); + speed.put("channel", DATAX_CHANNEL_COUNT); if (dataXParameters.getJobSpeedByte() > 0) { @@ -318,19 +332,20 @@ public class DataxTask extends AbstractTask { speed.put("record", dataXParameters.getJobSpeedRecord()); } - JSONObject errorLimit = new JSONObject(); + ObjectNode errorLimit = JSONUtils.createObjectNode(); errorLimit.put("record", 0); errorLimit.put("percentage", 0); - JSONObject setting = new JSONObject(); + ObjectNode setting = JSONUtils.createObjectNode(); setting.put("speed", speed); setting.put("errorLimit", errorLimit); return setting; } - private JSONObject buildDataxCoreJson() { - JSONObject speed = new JSONObject(); + private ObjectNode buildDataxCoreJson() { + + ObjectNode speed = JSONUtils.createObjectNode(); speed.put("channel", DATAX_CHANNEL_COUNT); if (dataXParameters.getJobSpeedByte() > 0) { @@ -341,26 +356,26 @@ public class DataxTask extends AbstractTask { speed.put("record", dataXParameters.getJobSpeedRecord()); } - JSONObject channel = new JSONObject(); - channel.put("speed", speed); + ObjectNode channel = JSONUtils.createObjectNode(); + channel.set("speed", speed); - JSONObject transport = new JSONObject(); - transport.put("channel", channel); + ObjectNode transport = JSONUtils.createObjectNode(); + transport.set("channel", channel); - JSONObject core = new JSONObject(); - core.put("transport", transport); + ObjectNode core = JSONUtils.createObjectNode(); + core.set("transport", transport); return core; } /** * create command - * + * * @return shell command file name * @throws Exception if error throws Exception */ private String buildShellCommandFile(String jobConfigFilePath) - throws Exception { + throws Exception { // generate scripts String fileName = String.format("%s/%s_node.sh", taskExecutionContext.getExecutePath(), @@ -411,7 +426,7 @@ public class DataxTask extends AbstractTask { /** * parsing synchronized column names in SQL statements - * + * * @param dsType * the database type of the data source * @param dtType @@ -437,7 +452,7 @@ public class DataxTask extends AbstractTask { /** * try grammatical parsing column - * + * * @param dbType * database type * @param sql @@ -467,7 +482,7 @@ public class DataxTask extends AbstractTask { } notNull(selectItemList, - String.format("select query type [%s] is not support", sqlSelect.getQuery().toString())); + String.format("select query type [%s] is not support", sqlSelect.getQuery().toString())); columnNames = new String[selectItemList.size()]; for (int i = 0; i < selectItemList.size(); i++ ) { @@ -487,12 +502,12 @@ public class DataxTask extends AbstractTask { } } else { throw new RuntimeException( - String.format("grammatical analysis sql column [ %s ] failed", item.toString())); + String.format("grammatical analysis sql column [ %s ] failed", item.toString())); } if (columnName == null) { throw new RuntimeException( - String.format("grammatical analysis sql column [ %s ] failed", item.toString())); + String.format("grammatical analysis sql column [ %s ] failed", item.toString())); } columnNames[i] = columnName; @@ -508,7 +523,7 @@ public class DataxTask extends AbstractTask { /** * try to execute sql to resolve column names - * + * * @param baseDataSource * the database connection parameters * @param sql @@ -521,10 +536,10 @@ public class DataxTask extends AbstractTask { sql = sql.replace(";", ""); try ( - Connection connection = DriverManager.getConnection(baseDataSource.getJdbcUrl(), baseDataSource.getUser(), - baseDataSource.getPassword()); - PreparedStatement stmt = connection.prepareStatement(sql); - ResultSet resultSet = stmt.executeQuery()) { + Connection connection = DriverManager.getConnection(baseDataSource.getJdbcUrl(), baseDataSource.getUser(), + baseDataSource.getPassword()); + PreparedStatement stmt = connection.prepareStatement(sql); + ResultSet resultSet = stmt.executeQuery()) { ResultSetMetaData md = resultSet.getMetaData(); int num = md.getColumnCount(); @@ -552,4 +567,4 @@ public class DataxTask extends AbstractTask { } } -} +} \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java index e506d28f35..39d6dbe767 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java @@ -17,8 +17,7 @@ package org.apache.dolphinscheduler.server.worker.task.http; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.io.Charsets; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; @@ -48,7 +47,6 @@ import org.slf4j.Logger; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -152,7 +150,7 @@ public class HttpTask extends AbstractTask { String jsonObject = JSONUtils.toJsonString(httpProperty); String params = ParameterUtils.convertParameterPlaceholders(jsonObject,ParamUtils.convert(paramsMap)); logger.info("http request params：{}",params); - httpPropertyList.add(JSON.parseObject(params,HttpProperty.class)); + httpPropertyList.add(JSONUtils.parseObject(params,HttpProperty.class)); } } addRequestParams(builder,httpPropertyList); @@ -252,7 +250,7 @@ public class HttpTask extends AbstractTask { */ protected void addRequestParams(RequestBuilder builder,List httpPropertyList) { if(CollectionUtils.isNotEmpty(httpPropertyList)){ - JSONObject jsonParam = new JSONObject(); + ObjectNode jsonParam = JSONUtils.createObjectNode(); for (HttpProperty property: httpPropertyList){ if(property.getHttpParametersType() != null){ if (property.getHttpParametersType().equals(HttpParametersType.PARAMETER)){ diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java index a90ca7a414..a11a8c0d45 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java @@ -16,8 +16,6 @@ */ package org.apache.dolphinscheduler.server.worker.task.processdure; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; import com.cronutils.utils.StringUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.*; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index 8a1d5ad352..3384ed17c7 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -16,9 +16,9 @@ */ package org.apache.dolphinscheduler.server.worker.task.sql; -import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; -import com.alibaba.fastjson.serializer.SerializerFeature; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.alert.utils.MailUtils; import org.apache.dolphinscheduler.common.Constants; @@ -43,6 +43,7 @@ import org.apache.dolphinscheduler.server.utils.UDFUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.slf4j.Logger; +import org.apache.dolphinscheduler.common.utils.*; import java.sql.*; import java.util.*; @@ -254,30 +255,30 @@ public class SqlTask extends AbstractTask { * @throws Exception */ private void resultProcess(ResultSet resultSet) throws Exception{ - JSONArray resultJSONArray = new JSONArray(); + ArrayNode resultJSONArray = JSONUtils.createArrayNode(); ResultSetMetaData md = resultSet.getMetaData(); int num = md.getColumnCount(); int rowCount = 0; while (rowCount < LIMIT && resultSet.next()) { - JSONObject mapOfColValues = new JSONObject(true); + ObjectNode mapOfColValues = JSONUtils.createObjectNode(); for (int i = 1; i <= num; i++) { - mapOfColValues.put(md.getColumnName(i), resultSet.getObject(i)); + mapOfColValues.set(md.getColumnName(i), JSONUtils.toJsonNode(resultSet.getObject(i))); } resultJSONArray.add(mapOfColValues); rowCount++; } - logger.debug("execute sql : {}", JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); + logger.debug("execute sql : {}", JSONUtils.toJsonString(resultJSONArray, SerializationFeature.WRITE_NULL_MAP_VALUES)); // if there is a result set - if (!resultJSONArray.isEmpty() ) { + if (!resultJSONArray.isEmpty(null) ) { if (StringUtils.isNotEmpty(sqlParameters.getTitle())) { sendAttachment(sqlParameters.getTitle(), - JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); + JSONUtils.toJsonString(resultJSONArray, SerializationFeature.WRITE_NULL_MAP_VALUES)); }else{ sendAttachment(taskExecutionContext.getTaskName() + " query resultsets ", - JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); + JSONUtils.toJsonString(resultJSONArray, SerializationFeature.WRITE_NULL_MAP_VALUES)); } } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java index 9f54d089be..c66fe95592 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.server.worker.task.sqoop; -import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; @@ -27,7 +26,9 @@ import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractYarnTask; import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.SqoopJobGenerator; import org.slf4j.Logger; + import java.util.Map; +import org.apache.dolphinscheduler.common.utils.*; /** * sqoop task extends the shell task @@ -50,7 +51,7 @@ public class SqoopTask extends AbstractYarnTask { public void init() throws Exception { logger.info("sqoop task params {}", taskExecutionContext.getTaskParams()); sqoopParameters = - JSON.parseObject(taskExecutionContext.getTaskParams(),SqoopParameters.class); + JSONUtils.parseObject(taskExecutionContext.getTaskParams(),SqoopParameters.class); if (!sqoopParameters.checkParameters()) { throw new RuntimeException("sqoop task params is not valid"); } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java index 70d91fd995..220cce5d18 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.utils; -import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.Direct; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index acc7a22ff0..ed3479ee0d 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -16,14 +16,12 @@ */ package org.apache.dolphinscheduler.server.worker.shell; -import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; -import org.apache.dolphinscheduler.server.worker.task.TaskManager; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -32,6 +30,7 @@ import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.dolphinscheduler.common.utils.*; import java.util.Date; @@ -68,7 +67,7 @@ public class ShellCommandExecutorTest { TaskInstance taskInstance = processService.findTaskInstanceById(7657); String taskJson = taskInstance.getTaskJson(); - TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class); + TaskNode taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); taskProps.setTaskParams(taskNode.getParams()); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java index 49301c3906..dbaa13215f 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java @@ -16,18 +16,17 @@ */ package org.apache.dolphinscheduler.server.worker.sql; -import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; -import org.apache.dolphinscheduler.server.worker.task.TaskManager; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -112,7 +111,7 @@ public class SqlExecutorTest { TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); String taskJson = taskInstance.getTaskJson(); - TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class); + TaskNode taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); taskProps.setTaskParams(taskNode.getParams()); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java index a2a46ef5a5..718ac27634 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java @@ -22,7 +22,8 @@ import java.util.Arrays; import java.util.Date; import java.util.List; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; @@ -289,24 +290,24 @@ public class DataxTaskTest { */ @Test public void testBuildDataxJobContentJson() - throws Exception { + throws Exception { try { Method method = DataxTask.class.getDeclaredMethod("buildDataxJobContentJson"); method.setAccessible(true); - List contentList = (List) method.invoke(dataxTask, null); + List contentList = (List) method.invoke(dataxTask, null); Assert.assertNotNull(contentList); - JSONObject content = contentList.get(0); - JSONObject reader = (JSONObject) content.get("reader"); + ObjectNode content = contentList.get(0); + JsonNode reader = content.path("reader"); Assert.assertNotNull(reader); - String readerPluginName = (String) reader.get("name"); + String readerPluginName = reader.path("name").asText(); Assert.assertEquals(DataxUtils.DATAX_READER_PLUGIN_MYSQL, readerPluginName); - JSONObject writer = (JSONObject) content.get("writer"); + JsonNode writer = content.path("writer"); Assert.assertNotNull(writer); - String writerPluginName = (String) writer.get("name"); + String writerPluginName = writer.path("name").asText(); Assert.assertEquals(DataxUtils.DATAX_WRITER_PLUGIN_MYSQL, writerPluginName); } catch (Exception e) { @@ -323,12 +324,11 @@ public class DataxTaskTest { try { Method method = DataxTask.class.getDeclaredMethod("buildDataxJobSettingJson"); method.setAccessible(true); - JSONObject setting = (JSONObject) method.invoke(dataxTask, null); + JsonNode setting = (JsonNode) method.invoke(dataxTask, null); Assert.assertNotNull(setting); Assert.assertNotNull(setting.get("speed")); Assert.assertNotNull(setting.get("errorLimit")); - } - catch (Exception e) { + } catch (Exception e) { Assert.fail(e.getMessage()); } } @@ -338,11 +338,11 @@ public class DataxTaskTest { */ @Test public void testBuildDataxCoreJson() - throws Exception { + throws Exception { try { Method method = DataxTask.class.getDeclaredMethod("buildDataxCoreJson"); method.setAccessible(true); - JSONObject coreConfig = (JSONObject) method.invoke(dataxTask, null); + ObjectNode coreConfig = (ObjectNode) method.invoke(dataxTask, null); Assert.assertNotNull(coreConfig); Assert.assertNotNull(coreConfig.get("transport")); } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java index bfc8205c2d..2af7bd34ef 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.server.worker.task.sqoop; -import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.dao.entity.DataSource; @@ -34,8 +33,9 @@ import org.mockito.junit.MockitoJUnitRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; +import org.apache.dolphinscheduler.common.utils.*; -import java.util.*; +import java.util.Date; /** * sqoop task test @@ -73,7 +73,7 @@ public class SqoopTaskTest { @Test public void testGenerator(){ String data1 = "{\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HDFS\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"0\\\",\\\"srcQuerySql\\\":\\\"\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[]}\",\"targetParams\":\"{\\\"targetPath\\\":\\\"/ods/tmp/test/person7\\\",\\\"deleteTargetDir\\\":true,\\\"fileType\\\":\\\"--as-textfile\\\",\\\"compressionCodec\\\":\\\"\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters1 = JSON.parseObject(data1,SqoopParameters.class); + SqoopParameters sqoopParameters1 = JSONUtils.parseObject(data1,SqoopParameters.class); SqoopJobGenerator generator = new SqoopJobGenerator(); String script = generator.generateSqoopJob(sqoopParameters1,new TaskExecutionContext()); @@ -81,21 +81,21 @@ public class SqoopTaskTest { Assert.assertEquals(expected, script); String data2 = "{\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HDFS\",\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"exportDir\\\":\\\"/ods/tmp/test/person7\\\"}\",\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"id,name,age,sex,create_time\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":true,\\\"targetUpdateKey\\\":\\\"id\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters2 = JSON.parseObject(data2,SqoopParameters.class); + SqoopParameters sqoopParameters2 = JSONUtils.parseObject(data2,SqoopParameters.class); String script2 = generator.generateSqoopJob(sqoopParameters2,new TaskExecutionContext()); String expected2 = "sqoop export -m 1 --export-dir /ods/tmp/test/person7 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --columns id,name,age,sex,create_time --fields-terminated-by '@' --lines-terminated-by '\\n' --update-key id --update-mode allowinsert"; Assert.assertEquals(expected2, script2); String data3 = "{\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HIVE\",\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-17\\\"}\",\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":false,\\\"targetUpdateKey\\\":\\\"\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters3 = JSON.parseObject(data3,SqoopParameters.class); + SqoopParameters sqoopParameters3 = JSONUtils.parseObject(data3,SqoopParameters.class); String script3 = generator.generateSqoopJob(sqoopParameters3,new TaskExecutionContext()); String expected3 = "sqoop export -m 1 --hcatalog-database stg --hcatalog-table person_internal --hcatalog-partition-keys date --hcatalog-partition-values 2020-02-17 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --fields-terminated-by '@' --lines-terminated-by '\\n'"; Assert.assertEquals(expected3, script3); String data4 = "{\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HIVE\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"1\\\",\\\"srcQuerySql\\\":\\\"SELECT * FROM person_2\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[{\\\"prop\\\":\\\"id\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"Integer\\\"}]}\",\"targetParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal_2\\\",\\\"createHiveTable\\\":true,\\\"dropDelimiter\\\":false,\\\"hiveOverWrite\\\":true,\\\"replaceDelimiter\\\":\\\"\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-16\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters4 = JSON.parseObject(data4,SqoopParameters.class); + SqoopParameters sqoopParameters4 = JSONUtils.parseObject(data4,SqoopParameters.class); String script4 = generator.generateSqoopJob(sqoopParameters4,new TaskExecutionContext()); String expected4 = "sqoop import -m 1 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --query 'SELECT * FROM person_2 WHERE $CONDITIONS' --map-column-java id=Integer --hive-import --hive-table stg.person_internal_2 --create-hive-table --hive-overwrite -delete-target-dir --hive-partition-key date --hive-partition-value 2020-02-16"; diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 5fef06d8b6..e8376c8b96 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -16,9 +16,8 @@ */ package org.apache.dolphinscheduler.service.process; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; import com.cronutils.model.Cron; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.lang.ArrayUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.*; @@ -205,16 +204,16 @@ public class ProcessService { CommandType commandType = command.getCommandType(); if(cmdTypeMap.containsKey(commandType)){ - JSONObject cmdParamObj = (JSONObject) JSON.parse(command.getCommandParam()); - JSONObject tempObj; - int processInstanceId = cmdParamObj.getInteger(CMDPARAM_RECOVER_PROCESS_ID_STRING); + ObjectNode cmdParamObj = JSONUtils.parseObject(command.getCommandParam()); + ObjectNode tempObj; + int processInstanceId = cmdParamObj.path(CMDPARAM_RECOVER_PROCESS_ID_STRING).asInt(); List commands = commandMapper.selectList(null); // for all commands for (Command tmpCommand:commands){ if(cmdTypeMap.containsKey(tmpCommand.getCommandType())){ - tempObj = (JSONObject) JSON.parse(tmpCommand.getCommandParam()); - if(tempObj != null && processInstanceId == tempObj.getInteger(CMDPARAM_RECOVER_PROCESS_ID_STRING)){ + tempObj = JSONUtils.parseObject(tmpCommand.getCommandParam()); + if(tempObj != null && processInstanceId == tempObj.path(CMDPARAM_RECOVER_PROCESS_ID_STRING).asInt()){ isNeedCreate = false; break; } From dea2d4634dee5eaf8d4f22c4376032b3d8b1268a Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 2 Jun 2020 20:13:23 +0800 Subject: [PATCH 07/38] Using Jackson instead of Fastjson --- .../apache/dolphinscheduler/common/utils/HadoopUtils.java | 2 +- .../dolphinscheduler/common/utils/ParameterUtils.java | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index c006dec6a8..360f4514b6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -406,7 +406,6 @@ public class HadoopUtils implements Closeable { * * @param applicationId application id * @return the return may be null or there may be other parse exceptions - * @throws JSONException json exception */ public ExecutionStatus getApplicationStatus(String applicationId) { if (StringUtils.isEmpty(applicationId)) { @@ -467,6 +466,7 @@ public class HadoopUtils implements Closeable { * hdfs resource dir * * @param tenantCode tenant code + * @param resourceType resource type * @return hdfs resource dir */ public static String getHdfsDir(ResourceType resourceType, String tenantCode) { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java index cae498340c..aae14a63ec 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java @@ -215,9 +215,10 @@ public class ParameterUtils { /** * new * $[yyyyMMdd] replace scheduler time - * @param text - * @param paramsMap - * @return + * @param text text + * @param scheduleTime scheduleTime + * @param paramsMap paramsMap + * @return text */ public static String replaceScheduleTime(String text, Date scheduleTime, Map paramsMap) { if (paramsMap != null) { From 33bd6586dd118acd29d8ab0eea155a6be5de3d7a Mon Sep 17 00:00:00 2001 From: simon Date: Wed, 3 Jun 2020 09:19:59 +0800 Subject: [PATCH 08/38] Using Jackson instead of Fastjson --- dolphinscheduler-remote/pom.xml | 4 ++ .../remote/command/TaskExecuteAckCommand.java | 2 +- .../command/TaskExecuteRequestCommand.java | 2 +- .../command/TaskExecuteResponseCommand.java | 2 +- .../command/TaskKillRequestCommand.java | 2 +- .../command/TaskKillResponseCommand.java | 2 +- .../log/GetLogBytesRequestCommand.java | 4 +- .../log/GetLogBytesResponseCommand.java | 4 +- .../log/RollViewLogRequestCommand.java | 4 +- .../log/RollViewLogResponseCommand.java | 4 +- .../command/log/ViewLogRequestCommand.java | 4 +- .../command/log/ViewLogResponseCommand.java | 4 +- ...Serializer.java => JacksonSerializer.java} | 39 ++++++++++++++++--- ...erTest.java => JacksonSerializerTest.java} | 8 ++-- .../server/entity/TaskExecutionContext.java | 4 +- .../server/log/LoggerRequestProcessor.java | 8 ++-- .../master/processor/TaskAckProcessor.java | 4 +- .../processor/TaskKillResponseProcessor.java | 4 +- .../processor/TaskResponseProcessor.java | 4 +- .../processor/TaskExecuteProcessor.java | 4 +- .../worker/processor/TaskKillProcessor.java | 4 +- .../service/log/LogClientService.java | 8 ++-- 22 files changed, 79 insertions(+), 46 deletions(-) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/{FastJsonSerializer.java => JacksonSerializer.java} (57%) rename dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/{FastJsonSerializerTest.java => JacksonSerializerTest.java} (84%) diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml index 0968e610bc..66994b4076 100644 --- a/dolphinscheduler-remote/pom.xml +++ b/dolphinscheduler-remote/pom.xml @@ -52,6 +52,10 @@ junit test + + com.fasterxml.jackson.core + jackson-databind + diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteAckCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteAckCommand.java index 0b3d901a3c..a965849ca3 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteAckCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteAckCommand.java @@ -1 +1 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.Date; /** * execute task request command */ public class TaskExecuteAckCommand implements Serializable { /** * taskInstanceId */ private int taskInstanceId; /** * startTime */ private Date startTime; /** * host */ private String host; /** * status */ private int status; /** * logPath */ private String logPath; /** * executePath */ private String executePath; public Date getStartTime() { return startTime; } public void setStartTime(Date startTime) { this.startTime = startTime; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } public String getLogPath() { return logPath; } public void setLogPath(String logPath) { this.logPath = logPath; } public String getExecutePath() { return executePath; } public void setExecutePath(String executePath) { this.executePath = executePath; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_EXECUTE_ACK); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskExecuteAckCommand{" + "taskInstanceId=" + taskInstanceId + ", startTime=" + startTime + ", host='" + host + '\'' + ", status=" + status + ", logPath='" + logPath + '\'' + ", executePath='" + executePath + '\'' + '}'; } } \ No newline at end of file +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import java.io.Serializable; import java.util.Date; /** * execute task request command */ public class TaskExecuteAckCommand implements Serializable { /** * taskInstanceId */ private int taskInstanceId; /** * startTime */ private Date startTime; /** * host */ private String host; /** * status */ private int status; /** * logPath */ private String logPath; /** * executePath */ private String executePath; public Date getStartTime() { return startTime; } public void setStartTime(Date startTime) { this.startTime = startTime; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } public String getLogPath() { return logPath; } public void setLogPath(String logPath) { this.logPath = logPath; } public String getExecutePath() { return executePath; } public void setExecutePath(String executePath) { this.executePath = executePath; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_EXECUTE_ACK); byte[] body = JacksonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskExecuteAckCommand{" + "taskInstanceId=" + taskInstanceId + ", startTime=" + startTime + ", host='" + host + '\'' + ", status=" + status + ", logPath='" + logPath + '\'' + ", executePath='" + executePath + '\'' + '}'; } } \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java index 637724f49d..affa17c08f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java @@ -1 +1 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; /** * execute task request command */ public class TaskExecuteRequestCommand implements Serializable { /** * task execution context */ private String taskExecutionContext; public String getTaskExecutionContext() { return taskExecutionContext; } public void setTaskExecutionContext(String taskExecutionContext) { this.taskExecutionContext = taskExecutionContext; } public TaskExecuteRequestCommand() { } public TaskExecuteRequestCommand(String taskExecutionContext) { this.taskExecutionContext = taskExecutionContext; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_EXECUTE_REQUEST); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskExecuteRequestCommand{" + "taskExecutionContext='" + taskExecutionContext + '\'' + '}'; } } \ No newline at end of file +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import java.io.Serializable; /** * execute task request command */ public class TaskExecuteRequestCommand implements Serializable { /** * task execution context */ private String taskExecutionContext; public String getTaskExecutionContext() { return taskExecutionContext; } public void setTaskExecutionContext(String taskExecutionContext) { this.taskExecutionContext = taskExecutionContext; } public TaskExecuteRequestCommand() { } public TaskExecuteRequestCommand(String taskExecutionContext) { this.taskExecutionContext = taskExecutionContext; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_EXECUTE_REQUEST); byte[] body = JacksonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskExecuteRequestCommand{" + "taskExecutionContext='" + taskExecutionContext + '\'' + '}'; } } \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java index deb6f5dd8f..4ac4bbb36a 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java @@ -1 +1 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.Date; /** * execute task response command */ public class TaskExecuteResponseCommand implements Serializable { public TaskExecuteResponseCommand() { } public TaskExecuteResponseCommand(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } /** * task instance id */ private int taskInstanceId; /** * status */ private int status; /** * end time */ private Date endTime; /** * processId */ private int processId; /** * appIds */ private String appIds; public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public Date getEndTime() { return endTime; } public void setEndTime(Date endTime) { this.endTime = endTime; } public int getProcessId() { return processId; } public void setProcessId(int processId) { this.processId = processId; } public String getAppIds() { return appIds; } public void setAppIds(String appIds) { this.appIds = appIds; } /** * package response command * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_EXECUTE_RESPONSE); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskExecuteResponseCommand{" + "taskInstanceId=" + taskInstanceId + ", status=" + status + ", endTime=" + endTime + ", processId=" + processId + ", appIds='" + appIds + '\'' + '}'; } } \ No newline at end of file +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import java.io.Serializable; import java.util.Date; /** * execute task response command */ public class TaskExecuteResponseCommand implements Serializable { public TaskExecuteResponseCommand() { } public TaskExecuteResponseCommand(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } /** * task instance id */ private int taskInstanceId; /** * status */ private int status; /** * end time */ private Date endTime; /** * processId */ private int processId; /** * appIds */ private String appIds; public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public Date getEndTime() { return endTime; } public void setEndTime(Date endTime) { this.endTime = endTime; } public int getProcessId() { return processId; } public void setProcessId(int processId) { this.processId = processId; } public String getAppIds() { return appIds; } public void setAppIds(String appIds) { this.appIds = appIds; } /** * package response command * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_EXECUTE_RESPONSE); byte[] body = JacksonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskExecuteResponseCommand{" + "taskInstanceId=" + taskInstanceId + ", status=" + status + ", endTime=" + endTime + ", processId=" + processId + ", appIds='" + appIds + '\'' + '}'; } } \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java index b8e02dd057..d1855e42eb 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java @@ -1 +1 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; /** * kill task request command */ public class TaskKillRequestCommand implements Serializable { /** * task id */ private int taskInstanceId; public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_KILL_REQUEST); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskKillRequestCommand{" + "taskInstanceId=" + taskInstanceId + '}'; } } \ No newline at end of file +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import java.io.Serializable; /** * kill task request command */ public class TaskKillRequestCommand implements Serializable { /** * task id */ private int taskInstanceId; public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_KILL_REQUEST); byte[] body = JacksonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskKillRequestCommand{" + "taskInstanceId=" + taskInstanceId + '}'; } } \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java index 2ca2330c80..2f35e58925 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java @@ -1 +1 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.Date; import java.util.List; /** * kill task response command */ public class TaskKillResponseCommand implements Serializable { /** * taskInstanceId */ private int taskInstanceId; /** * host */ private String host; /** * status */ private int status; /** * processId */ private int processId; /** * other resource manager appId , for example : YARN etc */ protected List appIds; public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public int getProcessId() { return processId; } public void setProcessId(int processId) { this.processId = processId; } public List getAppIds() { return appIds; } public void setAppIds(List appIds) { this.appIds = appIds; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_KILL_RESPONSE); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskKillResponseCommand{" + "taskInstanceId=" + taskInstanceId + ", host='" + host + '\'' + ", status=" + status + ", processId=" + processId + ", appIds=" + appIds + '}'; } } \ No newline at end of file +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import java.io.Serializable; import java.util.List; /** * kill task response command */ public class TaskKillResponseCommand implements Serializable { /** * taskInstanceId */ private int taskInstanceId; /** * host */ private String host; /** * status */ private int status; /** * processId */ private int processId; /** * other resource manager appId , for example : YARN etc */ protected List appIds; public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public int getProcessId() { return processId; } public void setProcessId(int processId) { this.processId = processId; } public List getAppIds() { return appIds; } public void setAppIds(List appIds) { this.appIds = appIds; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_KILL_RESPONSE); byte[] body = JacksonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskKillResponseCommand{" + "taskInstanceId=" + taskInstanceId + ", host='" + host + '\'' + ", status=" + status + ", processId=" + processId + ", appIds=" + appIds + '}'; } } \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java index 20cf8d9102..4dc9af486f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import java.io.Serializable; @@ -56,7 +56,7 @@ public class GetLogBytesRequestCommand implements Serializable { public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.GET_LOG_BYTES_REQUEST); - byte[] body = FastJsonSerializer.serialize(this); + byte[] body = JacksonSerializer.serialize(this); command.setBody(body); return command; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java index deaf9b8d85..32d1a5bea9 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import java.io.Serializable; @@ -57,7 +57,7 @@ public class GetLogBytesResponseCommand implements Serializable { public Command convert2Command(long opaque){ Command command = new Command(opaque); command.setType(CommandType.GET_LOG_BYTES_RESPONSE); - byte[] body = FastJsonSerializer.serialize(this); + byte[] body = JacksonSerializer.serialize(this); command.setBody(body); return command; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java index 433c4a0b1f..eeccc5860b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import java.io.Serializable; @@ -84,7 +84,7 @@ public class RollViewLogRequestCommand implements Serializable { public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.ROLL_VIEW_LOG_REQUEST); - byte[] body = FastJsonSerializer.serialize(this); + byte[] body = JacksonSerializer.serialize(this); command.setBody(body); return command; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java index 591d787200..dca2feed20 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import java.io.Serializable; @@ -57,7 +57,7 @@ public class RollViewLogResponseCommand implements Serializable { public Command convert2Command(long opaque){ Command command = new Command(opaque); command.setType(CommandType.ROLL_VIEW_LOG_RESPONSE); - byte[] body = FastJsonSerializer.serialize(this); + byte[] body = JacksonSerializer.serialize(this); command.setBody(body); return command; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java index b4773d0c0d..646d05c327 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import java.io.Serializable; @@ -56,7 +56,7 @@ public class ViewLogRequestCommand implements Serializable { public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.VIEW_WHOLE_LOG_REQUEST); - byte[] body = FastJsonSerializer.serialize(this); + byte[] body = JacksonSerializer.serialize(this); command.setBody(body); return command; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java index dffadade26..970d1c07b8 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import java.io.Serializable; @@ -57,7 +57,7 @@ public class ViewLogResponseCommand implements Serializable { public Command convert2Command(long opaque){ Command command = new Command(opaque); command.setType(CommandType.VIEW_WHOLE_LOG_RESPONSE); - byte[] body = FastJsonSerializer.serialize(this); + byte[] body = JacksonSerializer.serialize(this); command.setBody(body); return command; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/FastJsonSerializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JacksonSerializer.java similarity index 57% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/FastJsonSerializer.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JacksonSerializer.java index a18b8d5a7c..774510fe36 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/FastJsonSerializer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JacksonSerializer.java @@ -16,12 +16,20 @@ */ package org.apache.dolphinscheduler.remote.utils; -import com.alibaba.fastjson.JSON; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; /** * json serialize or deserialize */ -public class FastJsonSerializer { +public class JacksonSerializer { + private static final ObjectMapper objectMapper = new ObjectMapper(); + private static final Logger logger = LoggerFactory.getLogger(JacksonSerializer.class); /** * serialize to byte @@ -31,7 +39,13 @@ public class FastJsonSerializer { * @return byte array */ public static byte[] serialize(T obj) { - String json = JSON.toJSONString(obj); + String json = ""; + try { + json = objectMapper.writeValueAsString(obj); + } catch (JsonProcessingException e) { + logger.error("serializeToString exception!", e); + } + return json.getBytes(Constants.UTF8); } @@ -42,7 +56,14 @@ public class FastJsonSerializer { * @return string */ public static String serializeToString(T obj) { - return JSON.toJSONString(obj); + String json = ""; + try { + json = objectMapper.writeValueAsString(obj); + } catch (JsonProcessingException e) { + logger.error("serializeToString exception!", e); + } + + return json; } /** @@ -54,7 +75,15 @@ public class FastJsonSerializer { * @return deserialize type */ public static T deserialize(byte[] src, Class clazz) { - return JSON.parseObject(src, clazz); + + String json = new String(src, StandardCharsets.UTF_8); + try { + return objectMapper.readValue(json, clazz); + } catch (IOException e) { + logger.error("deserialize exception!", e); + return null; + } + } } diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/FastJsonSerializerTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/JacksonSerializerTest.java similarity index 84% rename from dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/FastJsonSerializerTest.java rename to dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/JacksonSerializerTest.java index 97166cca70..28ef0f98e2 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/FastJsonSerializerTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/JacksonSerializerTest.java @@ -18,20 +18,20 @@ package org.apache.dolphinscheduler.remote; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import org.junit.Assert; import org.junit.Test; -public class FastJsonSerializerTest { +public class JacksonSerializerTest { @Test public void testSerialize(){ TestObj testObj = new TestObj(); testObj.setAge(12); - byte[] serializeByte = FastJsonSerializer.serialize(testObj); + byte[] serializeByte = JacksonSerializer.serialize(testObj); // - TestObj deserialize = FastJsonSerializer.deserialize(serializeByte, TestObj.class); + TestObj deserialize = JacksonSerializer.deserialize(serializeByte, TestObj.class); Assert.assertEquals(testObj.getAge(), deserialize.getAge()); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java index 563f5c8459..148f04c649 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.entity; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import java.io.Serializable; import java.util.Date; @@ -431,7 +431,7 @@ public class TaskExecutionContext implements Serializable{ public Command toCommand(){ TaskExecuteRequestCommand requestCommand = new TaskExecuteRequestCommand(); - requestCommand.setTaskExecutionContext(FastJsonSerializer.serializeToString(this)); + requestCommand.setTaskExecutionContext(JacksonSerializer.serializeToString(this)); return requestCommand.convert2Command(); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java index 44ec68f89f..247597080a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java @@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.log.*; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,21 +61,21 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { final CommandType commandType = command.getType(); switch (commandType){ case GET_LOG_BYTES_REQUEST: - GetLogBytesRequestCommand getLogRequest = FastJsonSerializer.deserialize( + GetLogBytesRequestCommand getLogRequest = JacksonSerializer.deserialize( command.getBody(), GetLogBytesRequestCommand.class); byte[] bytes = getFileContentBytes(getLogRequest.getPath()); GetLogBytesResponseCommand getLogResponse = new GetLogBytesResponseCommand(bytes); channel.writeAndFlush(getLogResponse.convert2Command(command.getOpaque())); break; case VIEW_WHOLE_LOG_REQUEST: - ViewLogRequestCommand viewLogRequest = FastJsonSerializer.deserialize( + ViewLogRequestCommand viewLogRequest = JacksonSerializer.deserialize( command.getBody(), ViewLogRequestCommand.class); String msg = readWholeFileContent(viewLogRequest.getPath()); ViewLogResponseCommand viewLogResponse = new ViewLogResponseCommand(msg); channel.writeAndFlush(viewLogResponse.convert2Command(command.getOpaque())); break; case ROLL_VIEW_LOG_REQUEST: - RollViewLogRequestCommand rollViewLogRequest = FastJsonSerializer.deserialize( + RollViewLogRequestCommand rollViewLogRequest = JacksonSerializer.deserialize( command.getBody(), RollViewLogRequestCommand.class); List lines = readPartFileContent(rollViewLogRequest.getPath(), rollViewLogRequest.getSkipLineNum(), rollViewLogRequest.getLimit()); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java index 3460248dfb..b695310cae 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java @@ -28,7 +28,7 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.remote.utils.ChannelUtils; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import org.apache.dolphinscheduler.server.master.cache.TaskInstanceCacheManager; import org.apache.dolphinscheduler.server.master.cache.impl.TaskInstanceCacheManagerImpl; import org.apache.dolphinscheduler.server.master.processor.queue.TaskResponseEvent; @@ -77,7 +77,7 @@ public class TaskAckProcessor implements NettyRequestProcessor { @Override public void process(Channel channel, Command command) { Preconditions.checkArgument(CommandType.TASK_EXECUTE_ACK == command.getType(), String.format("invalid command type : %s", command.getType())); - TaskExecuteAckCommand taskAckCommand = FastJsonSerializer.deserialize(command.getBody(), TaskExecuteAckCommand.class); + TaskExecuteAckCommand taskAckCommand = JacksonSerializer.deserialize(command.getBody(), TaskExecuteAckCommand.class); logger.info("taskAckCommand : {}", taskAckCommand); taskInstanceCacheManager.cacheTaskInstance(taskAckCommand); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java index 3e8cdfdadc..720f8fd751 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java @@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskKillResponseCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +45,7 @@ public class TaskKillResponseProcessor implements NettyRequestProcessor { public void process(Channel channel, Command command) { Preconditions.checkArgument(CommandType.TASK_KILL_RESPONSE == command.getType(), String.format("invalid command type : %s", command.getType())); - TaskKillResponseCommand responseCommand = FastJsonSerializer.deserialize(command.getBody(), TaskKillResponseCommand.class); + TaskKillResponseCommand responseCommand = JacksonSerializer.deserialize(command.getBody(), TaskKillResponseCommand.class); logger.info("received task kill response command : {}", responseCommand); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java index 721b146d86..0d1214ca2b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java @@ -27,7 +27,7 @@ import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import org.apache.dolphinscheduler.server.master.cache.TaskInstanceCacheManager; import org.apache.dolphinscheduler.server.master.cache.impl.TaskInstanceCacheManagerImpl; import org.apache.dolphinscheduler.server.master.processor.queue.TaskResponseEvent; @@ -78,7 +78,7 @@ public class TaskResponseProcessor implements NettyRequestProcessor { public void process(Channel channel, Command command) { Preconditions.checkArgument(CommandType.TASK_EXECUTE_RESPONSE == command.getType(), String.format("invalid command type : %s", command.getType())); - TaskExecuteResponseCommand responseCommand = FastJsonSerializer.deserialize(command.getBody(), TaskExecuteResponseCommand.class); + TaskExecuteResponseCommand responseCommand = JacksonSerializer.deserialize(command.getBody(), TaskExecuteResponseCommand.class); logger.info("received command : {}", responseCommand); taskInstanceCacheManager.cacheTaskInstance(responseCommand); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java index 79740a2d08..2009c40cd7 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java @@ -31,7 +31,7 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; @@ -78,7 +78,7 @@ public class TaskExecuteProcessor implements NettyRequestProcessor { Preconditions.checkArgument(CommandType.TASK_EXECUTE_REQUEST == command.getType(), String.format("invalid command type : %s", command.getType())); - TaskExecuteRequestCommand taskRequestCommand = FastJsonSerializer.deserialize( + TaskExecuteRequestCommand taskRequestCommand = JacksonSerializer.deserialize( command.getBody(), TaskExecuteRequestCommand.class); logger.info("received command : {}", taskRequestCommand); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java index b6f58279b1..76323f97a8 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java @@ -29,7 +29,7 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskKillRequestCommand; import org.apache.dolphinscheduler.remote.command.TaskKillResponseCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.Pair; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; @@ -83,7 +83,7 @@ public class TaskKillProcessor implements NettyRequestProcessor { @Override public void process(Channel channel, Command command) { Preconditions.checkArgument(CommandType.TASK_KILL_REQUEST == command.getType(), String.format("invalid command type : %s", command.getType())); - TaskKillRequestCommand killCommand = FastJsonSerializer.deserialize(command.getBody(), TaskKillRequestCommand.class); + TaskKillRequestCommand killCommand = JacksonSerializer.deserialize(command.getBody(), TaskKillRequestCommand.class); logger.info("received kill command : {}", killCommand); Pair> result = doKill(killCommand); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java index 8e63c89405..c1a766d531 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java @@ -21,7 +21,7 @@ import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.log.*; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,7 +77,7 @@ public class LogClientService { Command command = request.convert2Command(); Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); if(response != null){ - RollViewLogResponseCommand rollReviewLog = FastJsonSerializer.deserialize( + RollViewLogResponseCommand rollReviewLog = JacksonSerializer.deserialize( response.getBody(), RollViewLogResponseCommand.class); return rollReviewLog.getMsg(); } @@ -105,7 +105,7 @@ public class LogClientService { Command command = request.convert2Command(); Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); if(response != null){ - ViewLogResponseCommand viewLog = FastJsonSerializer.deserialize( + ViewLogResponseCommand viewLog = JacksonSerializer.deserialize( response.getBody(), ViewLogResponseCommand.class); return viewLog.getMsg(); } @@ -133,7 +133,7 @@ public class LogClientService { Command command = request.convert2Command(); Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); if(response != null){ - GetLogBytesResponseCommand getLog = FastJsonSerializer.deserialize( + GetLogBytesResponseCommand getLog = JacksonSerializer.deserialize( response.getBody(), GetLogBytesResponseCommand.class); return getLog.getData(); } From bb089288c4fe8e228d119737dfd10c2b21f7328b Mon Sep 17 00:00:00 2001 From: simon Date: Wed, 3 Jun 2020 12:40:28 +0800 Subject: [PATCH 09/38] Using Jackson instead of Fastjson --- .../alert/utils/JSONUtilsTest.java | 117 ------------------ .../common/utils/JSONUtils.java | 14 --- .../common/utils/JSONUtilsTest.java | 79 ++++++++---- 3 files changed, 58 insertions(+), 152 deletions(-) delete mode 100644 dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java deleted file mode 100644 index fdd98b87c7..0000000000 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.alert.utils; - -import com.fasterxml.jackson.databind.JsonNode; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; - -import org.apache.dolphinscheduler.common.utils.*; - -import static org.junit.Assert.*; - -public class JSONUtilsTest { - - private static final Logger logger = LoggerFactory.getLogger(JSONUtilsTest.class); - - public List> list = new ArrayList<>(); - - public String expected = null; - - @Before - public void setUp() throws Exception { - - //Define expected json string - expected = "[{\"mysql service name\":\"mysql200\",\"mysql address\":\"192.168.xx.xx\",\"port\":\"3306\",\"no index of number\":\"80\",\"database client connections\":\"190\"}]"; - - //Initial map - LinkedHashMap map = new LinkedHashMap<>(); - map.put("mysql service name", "mysql200"); - map.put("mysql address", "192.168.xx.xx"); - map.put("port", "3306"); - map.put("no index of number", "80"); - map.put("database client connections", "190"); - - //Add map into list - list.add(map); - } - - - /** - * Test toJsonString - */ - @Test - public void testToJsonString() { - - //Invoke toJsonString - String result = JSONUtils.toJsonString(list); - logger.info(result); - - //Equal result with expected string - assertEquals(result, expected); -// assertEquals(result, "[{\"database client connections\":\"190\",\"mysql address\":\"192.168.xx.xx\",\"mysql service name\":\"mysql200\",\"no index of number\":\"80\",\"port\":\"3306\"}]"); - - //If param is null, then return null string - result = JSONUtils.toJsonString(null); - logger.info(result); - - assertEquals("null", result); - - } - - /** - * Test toList - */ - @Test - public void testToList() { - - //Invoke toList - List result = JSONUtils.toList(expected, LinkedHashMap.class); - //Equal list size=1 - assertEquals(1, result.size()); - - //Transform entity to LinkedHashMap - LinkedHashMap entity = result.get(0); - - //Equal expected values - assertEquals("mysql200", entity.get("mysql service name")); - assertEquals("192.168.xx.xx", entity.get("mysql address")); - assertEquals("3306", entity.get("port")); - assertEquals("80", entity.get("no index of number")); - assertEquals("190", entity.get("database client connections")); - - //If param is null, then return empty list - result = JSONUtils.toList(null, LinkedHashMap.class); - assertNotNull(result); - assertTrue(result.isEmpty()); - - //If param is incorrect, then return empty list and log error message - result = JSONUtils.toList("}{", LinkedHashMap.class); - assertNotNull(result); - assertTrue(result.isEmpty()); - - } - - -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index a4c6093157..7e3b119502 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -108,20 +108,6 @@ public class JSONUtils { return null; } - - public static T parseObject(String json, TypeReference typeReference) { - if (StringUtils.isEmpty(json)) { - return null; - } - - try { - return objectMapper.readValue(json, typeReference); - } catch (Exception e) { - logger.error("parse object exception!", e); - } - return null; - } - /** * json to list * diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java index c62633d1dc..671b8d11b7 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -29,20 +30,66 @@ import java.util.*; public class JSONUtilsTest { + @Test + public void createArrayNodeTest() { + Property property = new Property(); + property.setProp("ds"); + property.setDirect(Direct.IN); + property.setType(DataType.VARCHAR); + property.setValue("sssssss"); + String str = "[{\"prop\":\"ds\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"},{\"prop\":\"ds\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"}]"; + JsonNode jsonNode = JSONUtils.toJsonNode(property); + + ArrayNode arrayNode = JSONUtils.createArrayNode(); + ArrayList objects = new ArrayList<>(); + objects.add(jsonNode); + objects.add(jsonNode); + + ArrayNode jsonNodes = arrayNode.addAll(objects); + String s = JSONUtils.toJsonString(jsonNodes); + Assert.assertEquals(s, str); + + } + + @Test + public void toJsonNodeTest() { + Property property = new Property(); + property.setProp("ds"); + property.setDirect(Direct.IN); + property.setType(DataType.VARCHAR); + property.setValue("sssssss"); + String str = "{\"prop\":\"ds\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"}"; + + JsonNode jsonNodes = JSONUtils.toJsonNode(property); + String s = JSONUtils.toJsonString(jsonNodes); + Assert.assertEquals(s, str); + + } + + @Test + public void createObjectNodeTest() { + String jsonStr = "{\"a\":\"b\",\"b\":\"d\"}"; + + ObjectNode objectNode = JSONUtils.createObjectNode(); + objectNode.put("a","b"); + objectNode.put("b","d"); + String s = JSONUtils.toJsonString(objectNode); + Assert.assertEquals(s, jsonStr); + } @Test public void toMap() { String jsonStr = "{\"id\":\"1001\",\"name\":\"Jobs\"}"; - Map models = JSONUtils.toMap(jsonStr); + Map models = JSONUtils.toMap(jsonStr); Assert.assertEquals("1001", models.get("id")); Assert.assertEquals("Jobs", models.get("name")); } @Test - public void convert2Property(){ + public void convert2Property() { Property property = new Property(); property.setProp("ds"); property.setDirect(Direct.IN); @@ -56,7 +103,7 @@ public class JSONUtilsTest { @Test - public void String2MapTest(){ + public void String2MapTest() { String str = list2String(); List maps = JSONUtils.toList(str, @@ -70,31 +117,21 @@ public class JSONUtilsTest { Assert.assertEquals("190", maps.get(0).get("database client connections")); } - public String list2String(){ + public String list2String() { LinkedHashMap map1 = new LinkedHashMap<>(); - map1.put("mysql service name","mysql200"); - map1.put("mysql address","192.168.xx.xx"); - map1.put("port","3306"); - map1.put("no index of number","80"); - map1.put("database client connections","190"); + map1.put("mysql service name", "mysql200"); + map1.put("mysql address", "192.168.xx.xx"); + map1.put("port", "3306"); + map1.put("no index of number", "80"); + map1.put("database client connections", "190"); List> maps = new ArrayList<>(); - maps.add(0,map1); + maps.add(0, map1); String resultJson = JSONUtils.toJsonString(maps); return resultJson; } - @Test - public void testToJson() { - Map map = new HashMap<>(); - map.put("foo","bar"); - - Assert.assertEquals("{\"foo\":\"bar\"}", JSONUtils.toJsonString(map)); - Assert.assertEquals( - String.valueOf((Object) null), JSONUtils.toJsonString(null)); - } - @Test public void testParseObject() { Assert.assertNull(JSONUtils.parseObject("")); @@ -124,7 +161,7 @@ public class JSONUtilsTest { @Test public void testToMap() { Map map = new HashMap<>(); - map.put("foo","bar"); + map.put("foo", "bar"); Assert.assertTrue(map.equals(JSONUtils.toMap( "{\n" + "\"foo\": \"bar\"\n" + "}"))); From 8bab8abcd23cd6a03d8b567901da7b5959566f69 Mon Sep 17 00:00:00 2001 From: simon Date: Wed, 3 Jun 2020 19:21:19 +0800 Subject: [PATCH 10/38] ut fix --- .../common/utils/JSONUtilsTest.java | 4 + .../worker/task/datax/DataxTaskTest.java | 100 ++++++++++-------- 2 files changed, 60 insertions(+), 44 deletions(-) diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java index 671b8d11b7..eca392d9e2 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.common.utils; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -188,6 +189,9 @@ public class JSONUtilsTest { JSONUtils.toJsonString(map)); Assert.assertEquals(String.valueOf((Object) null), JSONUtils.toJsonString(null)); + + Assert.assertEquals("{\"foo\":\"bar\"}", + JSONUtils.toJsonString(map, SerializationFeature.WRITE_NULL_MAP_VALUES)); } @Test diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java index 718ac27634..74da998a03 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java @@ -26,6 +26,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.task.datax.DataxParameters; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; import org.apache.dolphinscheduler.dao.entity.DataSource; @@ -71,7 +73,12 @@ public class DataxTaskTest { @Before public void before() - throws Exception { + throws Exception { + setTaskParems(0); + } + + private void setTaskParems(Integer customConfig) { + processService = Mockito.mock(ProcessService.class); shellCommandExecutor = Mockito.mock(ShellCommandExecutor.class); @@ -88,8 +95,15 @@ public class DataxTaskTest { props.setEnvFile(".dolphinscheduler_env.sh"); props.setTaskStartTime(new Date()); props.setTaskTimeout(0); - props.setTaskParams( - "{\"targetTable\":\"test\",\"postStatements\":[],\"jobSpeedByte\":1024,\"jobSpeedRecord\":1000,\"dtType\":\"MYSQL\",\"datasource\":1,\"dsType\":\"MYSQL\",\"datatarget\":2,\"jobSpeedByte\":0,\"sql\":\"select 1 as test from dual\",\"preStatements\":[\"delete from test\"],\"postStatements\":[\"delete from test\"]}"); + if (customConfig == 1) { + props.setTaskParams( + "{\"customConfig\":1, \"localParams\":[{\"prop\":\"test\",\"value\":\"38294729\"}],\"json\":\"{\\\"job\\\":{\\\"setting\\\":{\\\"speed\\\":{\\\"byte\\\":1048576},\\\"errorLimit\\\":{\\\"record\\\":0,\\\"percentage\\\":0.02}},\\\"content\\\":[{\\\"reader\\\":{\\\"name\\\":\\\"rdbmsreader\\\",\\\"parameter\\\":{\\\"username\\\":\\\"xxx\\\",\\\"password\\\":\\\"${test}\\\",\\\"column\\\":[\\\"id\\\",\\\"name\\\"],\\\"splitPk\\\":\\\"pk\\\",\\\"connection\\\":[{\\\"querySql\\\":[\\\"SELECT * from dual\\\"],\\\"jdbcUrl\\\":[\\\"jdbc:dm://ip:port/database\\\"]}],\\\"fetchSize\\\":1024,\\\"where\\\":\\\"1 = 1\\\"}},\\\"writer\\\":{\\\"name\\\":\\\"streamwriter\\\",\\\"parameter\\\":{\\\"print\\\":true}}}]}}\"}"); + +// "{\"customConfig\":1,\"json\":\"{\\\"job\\\":{\\\"setting\\\":{\\\"speed\\\":{\\\"byte\\\":1048576},\\\"errorLimit\\\":{\\\"record\\\":0,\\\"percentage\\\":0.02}},\\\"content\\\":[{\\\"reader\\\":{\\\"name\\\":\\\"rdbmsreader\\\",\\\"parameter\\\":{\\\"username\\\":\\\"xxx\\\",\\\"password\\\":\\\"xxx\\\",\\\"column\\\":[\\\"id\\\",\\\"name\\\"],\\\"splitPk\\\":\\\"pk\\\",\\\"connection\\\":[{\\\"querySql\\\":[\\\"SELECT * from dual\\\"],\\\"jdbcUrl\\\":[\\\"jdbc:dm://ip:port/database\\\"]}],\\\"fetchSize\\\":1024,\\\"where\\\":\\\"1 = 1\\\"}},\\\"writer\\\":{\\\"name\\\":\\\"streamwriter\\\",\\\"parameter\\\":{\\\"print\\\":true}}}]}}\"}"); + } else { + props.setTaskParams( + "{\"customConfig\":0,\"targetTable\":\"test\",\"postStatements\":[],\"jobSpeedByte\":1024,\"jobSpeedRecord\":1000,\"dtType\":\"MYSQL\",\"dataSource\":1,\"dsType\":\"MYSQL\",\"dataTarget\":2,\"jobSpeedByte\":0,\"sql\":\"select 1 as test from dual\",\"preStatements\":[\"delete from test\"],\"postStatements\":[\"delete from test\"]}"); + } taskExecutionContext = Mockito.mock(TaskExecutionContext.class); Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(props.getTaskParams()); @@ -111,28 +125,20 @@ public class DataxTaskTest { dataxTask = PowerMockito.spy(new DataxTask(taskExecutionContext, logger)); dataxTask.init(); props.setCmdTypeIfComplement(START_PROCESS); - setTaskParems(0); + Mockito.when(processService.findDataSourceById(1)).thenReturn(getDataSource()); Mockito.when(processService.findDataSourceById(2)).thenReturn(getDataSource()); Mockito.when(processService.findProcessInstanceByTaskId(1)).thenReturn(getProcessInstance()); String fileName = String.format("%s/%s_node.sh", props.getExecutePath(), props.getTaskAppId()); - Mockito.when(shellCommandExecutor.run(fileName)).thenReturn(null); - } - - private void setTaskParems(Integer customConfig) { - if (customConfig == 1) { - props.setTaskParams( - "{\"customConfig\":1, \"localParams\":[{\"prop\":\"test\",\"value\":\"38294729\"}],\"json\":\"{\\\"job\\\":{\\\"setting\\\":{\\\"speed\\\":{\\\"byte\\\":1048576},\\\"errorLimit\\\":{\\\"record\\\":0,\\\"percentage\\\":0.02}},\\\"content\\\":[{\\\"reader\\\":{\\\"name\\\":\\\"rdbmsreader\\\",\\\"parameter\\\":{\\\"username\\\":\\\"xxx\\\",\\\"password\\\":\\\"${test}\\\",\\\"column\\\":[\\\"id\\\",\\\"name\\\"],\\\"splitPk\\\":\\\"pk\\\",\\\"connection\\\":[{\\\"querySql\\\":[\\\"SELECT * from dual\\\"],\\\"jdbcUrl\\\":[\\\"jdbc:dm://ip:port/database\\\"]}],\\\"fetchSize\\\":1024,\\\"where\\\":\\\"1 = 1\\\"}},\\\"writer\\\":{\\\"name\\\":\\\"streamwriter\\\",\\\"parameter\\\":{\\\"print\\\":true}}}]}}\"}"); - -// "{\"customConfig\":1,\"json\":\"{\\\"job\\\":{\\\"setting\\\":{\\\"speed\\\":{\\\"byte\\\":1048576},\\\"errorLimit\\\":{\\\"record\\\":0,\\\"percentage\\\":0.02}},\\\"content\\\":[{\\\"reader\\\":{\\\"name\\\":\\\"rdbmsreader\\\",\\\"parameter\\\":{\\\"username\\\":\\\"xxx\\\",\\\"password\\\":\\\"xxx\\\",\\\"column\\\":[\\\"id\\\",\\\"name\\\"],\\\"splitPk\\\":\\\"pk\\\",\\\"connection\\\":[{\\\"querySql\\\":[\\\"SELECT * from dual\\\"],\\\"jdbcUrl\\\":[\\\"jdbc:dm://ip:port/database\\\"]}],\\\"fetchSize\\\":1024,\\\"where\\\":\\\"1 = 1\\\"}},\\\"writer\\\":{\\\"name\\\":\\\"streamwriter\\\",\\\"parameter\\\":{\\\"print\\\":true}}}]}}\"}"); - } else { - props.setTaskParams( - "{\"customConfig\":0,\"targetTable\":\"test\",\"postStatements\":[],\"jobSpeedByte\":1024,\"jobSpeedRecord\":1000,\"dtType\":\"MYSQL\",\"datasource\":1,\"dsType\":\"MYSQL\",\"datatarget\":2,\"jobSpeedByte\":0,\"sql\":\"select 1 as test from dual\",\"preStatements\":[\"delete from test\"],\"postStatements\":[\"delete from test\"]}"); - + try { + Mockito.when(shellCommandExecutor.run(fileName)).thenReturn(null); + } catch (Exception e) { + e.printStackTrace(); } + dataxTask = PowerMockito.spy(new DataxTask(taskExecutionContext, logger)); dataxTask.init(); } @@ -154,7 +160,8 @@ public class DataxTaskTest { @After public void after() - throws Exception {} + throws Exception { + } /** * Method: DataxTask() @@ -210,7 +217,7 @@ public class DataxTaskTest { */ @Test public void testParsingSqlColumnNames() - throws Exception { + throws Exception { try { BaseDataSource dataSource = DataSourceFactory.getDatasource(getDataSource().getType(), getDataSource().getConnectionParams()); @@ -224,8 +231,7 @@ public class DataxTaskTest { Assert.assertTrue(columns.length == 2); Assert.assertEquals("[`a`, `table`]", Arrays.toString(columns)); - } - catch (Exception e) { + } catch (Exception e) { Assert.fail(e.getMessage()); } } @@ -235,7 +241,7 @@ public class DataxTaskTest { */ @Test public void testTryGrammaticalAnalysisSqlColumnNames() - throws Exception { + throws Exception { try { Method method = DataxTask.class.getDeclaredMethod("tryGrammaticalAnalysisSqlColumnNames", DbType.class, String.class); method.setAccessible(true); @@ -246,8 +252,7 @@ public class DataxTaskTest { Assert.assertTrue(columns.length == 2); Assert.assertEquals("[a, b]", Arrays.toString(columns)); - } - catch (Exception e) { + } catch (Exception e) { Assert.fail(e.getMessage()); } } @@ -258,7 +263,7 @@ public class DataxTaskTest { */ @Test public void testTryExecuteSqlResolveColumnNames() - throws Exception { + throws Exception { // TODO: Test goes here... } @@ -270,19 +275,30 @@ public class DataxTaskTest { throws Exception { try { setTaskParems(1); - buildDataJson(); - setTaskParems(0); - buildDataJson(); + Method method = DataxTask.class.getDeclaredMethod("buildDataxJsonFile"); + method.setAccessible(true); + String filePath = (String) method.invoke(dataxTask, null); + Assert.assertNotNull(filePath); } catch (Exception e) { Assert.fail(e.getMessage()); } } - public void buildDataJson() throws Exception { - Method method = DataxTask.class.getDeclaredMethod("buildDataxJsonFile"); - method.setAccessible(true); - String filePath = (String) method.invoke(dataxTask, null); - Assert.assertNotNull(filePath); + /** + * Method: buildDataxJsonFile() + */ + @Test + public void testBuildDataxJsonFile0() + throws Exception { + try { + setTaskParems(0); + Method method = DataxTask.class.getDeclaredMethod("buildDataxJsonFile"); + method.setAccessible(true); + String filePath = (String) method.invoke(dataxTask, null); + Assert.assertNotNull(filePath); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } } /** @@ -309,8 +325,7 @@ public class DataxTaskTest { String writerPluginName = writer.path("name").asText(); Assert.assertEquals(DataxUtils.DATAX_WRITER_PLUGIN_MYSQL, writerPluginName); - } - catch (Exception e) { + } catch (Exception e) { Assert.fail(e.getMessage()); } } @@ -320,7 +335,7 @@ public class DataxTaskTest { */ @Test public void testBuildDataxJobSettingJson() - throws Exception { + throws Exception { try { Method method = DataxTask.class.getDeclaredMethod("buildDataxJobSettingJson"); method.setAccessible(true); @@ -345,8 +360,7 @@ public class DataxTaskTest { ObjectNode coreConfig = (ObjectNode) method.invoke(dataxTask, null); Assert.assertNotNull(coreConfig); Assert.assertNotNull(coreConfig.get("transport")); - } - catch (Exception e) { + } catch (Exception e) { Assert.fail(e.getMessage()); } } @@ -356,13 +370,12 @@ public class DataxTaskTest { */ @Test public void testBuildShellCommandFile() - throws Exception { + throws Exception { try { Method method = DataxTask.class.getDeclaredMethod("buildShellCommandFile", String.class); method.setAccessible(true); Assert.assertNotNull(method.invoke(dataxTask, "test.json")); - } - catch (Exception e) { + } catch (Exception e) { Assert.fail(e.getMessage()); } } @@ -381,13 +394,12 @@ public class DataxTaskTest { */ @Test public void testNotNull() - throws Exception { + throws Exception { try { Method method = DataxTask.class.getDeclaredMethod("notNull", Object.class, String.class); method.setAccessible(true); method.invoke(dataxTask, "abc", "test throw RuntimeException"); - } - catch (Exception e) { + } catch (Exception e) { Assert.fail(e.getMessage()); } } From 9c36829278b4e5ff2259bfe0014c19e5345e349d Mon Sep 17 00:00:00 2001 From: simon Date: Wed, 3 Jun 2020 20:05:07 +0800 Subject: [PATCH 11/38] ut fix --- .../server/worker/task/datax/DataxTaskTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java index 74da998a03..5a9ff568ae 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java @@ -21,6 +21,7 @@ import java.lang.reflect.Method; import java.util.Arrays; import java.util.Date; import java.util.List; +import java.util.UUID; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -108,7 +109,7 @@ public class DataxTaskTest { taskExecutionContext = Mockito.mock(TaskExecutionContext.class); Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(props.getTaskParams()); Mockito.when(taskExecutionContext.getExecutePath()).thenReturn("/tmp"); - Mockito.when(taskExecutionContext.getTaskAppId()).thenReturn("1"); + Mockito.when(taskExecutionContext.getTaskAppId()).thenReturn(UUID.randomUUID().toString()); Mockito.when(taskExecutionContext.getTenantCode()).thenReturn("root"); Mockito.when(taskExecutionContext.getStartTime()).thenReturn(new Date()); Mockito.when(taskExecutionContext.getTaskTimeout()).thenReturn(10000); @@ -273,6 +274,7 @@ public class DataxTaskTest { @Test public void testBuildDataxJsonFile() throws Exception { + try { setTaskParems(1); Method method = DataxTask.class.getDeclaredMethod("buildDataxJsonFile"); From 4a10157506c0659e6bff20fdb47b930a4630b1c1 Mon Sep 17 00:00:00 2001 From: simon Date: Thu, 4 Jun 2020 09:42:41 +0800 Subject: [PATCH 12/38] ut fix --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5dca76ce4c..7c58ebc373 100644 --- a/pom.xml +++ b/pom.xml @@ -786,7 +786,7 @@ **/dao/mapper/CommandMapperTest.java **/dao/mapper/ConnectionFactoryTest.java **/dao/mapper/DataSourceMapperTest.java - **/remote/FastJsonSerializerTest.java + **/remote/JacksonSerializerTest.java **/remote/NettyRemotingClientTest.java **/remote/ResponseFutureTest.java **/server/log/MasterLogFilterTest.java @@ -821,6 +821,7 @@ **/server/worker/task/spark/SparkTaskTest.java **/server/worker/task/EnvFileTest.java **/server/worker/task/spark/SparkTaskTest.java + **/server/worker/task/datax/DataxTaskTest.java **/server/worker/EnvFileTest.java **/service/quartz/cron/CronUtilsTest.java **/service/zk/DefaultEnsembleProviderTest.java From 6205e687c2c96bf0a87e300081d1395e509b26c3 Mon Sep 17 00:00:00 2001 From: simon Date: Thu, 4 Jun 2020 10:11:11 +0800 Subject: [PATCH 13/38] annotation reset --- .../api/dto/resources/ResourceComponent.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java index 6a5b6e3353..3f7c227ae0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java @@ -1,6 +1,7 @@ package org.apache.dolphinscheduler.api.dto.resources; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.alibaba.fastjson.annotation.JSONType; +import com.alibaba.fastjson.annotation.JSONField; import org.apache.dolphinscheduler.common.enums.ResourceType; import java.util.ArrayList; @@ -25,7 +26,7 @@ import java.util.List; /** * resource component */ -@JsonPropertyOrder({"id","pid","name","fullName","description","isDirctory","children","type"}) +@JSONType(orders={"id","pid","name","fullName","description","isDirctory","children","type"}) public abstract class ResourceComponent { public ResourceComponent() { } @@ -45,14 +46,17 @@ public abstract class ResourceComponent { /** * id */ + @JSONField(ordinal = 1) protected int id; /** * parent id */ + @JSONField(ordinal = 2) protected int pid; /** * name */ + @JSONField(ordinal = 3) protected String name; /** * current directory @@ -61,26 +65,32 @@ public abstract class ResourceComponent { /** * full name */ + @JSONField(ordinal = 4) protected String fullName; /** * description */ + @JSONField(ordinal = 5) protected String description; /** * is directory */ + @JSONField(ordinal = 6) protected boolean isDirctory; /** * id value */ + @JSONField(ordinal = 7) protected String idValue; /** * resoruce type */ + @JSONField(ordinal = 8) protected ResourceType type; /** * children */ + @JSONField(ordinal = 8) protected List children = new ArrayList<>(); /** From 572ee0fc6474bc238761eca79848e8f12fff22c8 Mon Sep 17 00:00:00 2001 From: simon824 Date: Thu, 4 Jun 2020 16:23:59 +0800 Subject: [PATCH 14/38] Using Jackson instead of Fastjson --- .../api/service/ProcessDefinitionService.java | 8 +++- .../api/service/ProcessInstanceService.java | 21 ++++++++-- .../common/utils/JSONUtils.java | 7 +++- .../dao/entity/ProcessDefinition.java | 34 ++++++++++++---- .../dao/entity/ProcessDefinitionTest.java | 39 +++++++++++++++++++ .../worker/runner/TaskExecuteThread.java | 9 ++++- .../service/process/ProcessService.java | 14 ++++++- 7 files changed, 116 insertions(+), 16 deletions(-) create mode 100644 dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index be646ddbaa..b76bfd00e3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.api.service; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -755,8 +756,13 @@ public class ProcessDefinitionService extends BaseDAGService { public Map importProcessDefinition(User loginUser, MultipartFile file, String currentProjectName) { Map result = new HashMap<>(5); String processMetaJson = FileUtils.file2String(file); - List processMetaList = JSONUtils.toList(processMetaJson, ProcessMeta.class); + List processMetaList = new ArrayList<>(); + try { + processMetaList = JSONUtils.getMapper().readValue(processMetaJson, new TypeReference>() {}); + } catch (Exception e) { + logger.error("parse list exception!", e); + } //check file content if (CollectionUtils.isEmpty(processMetaList)) { putMsg(result, Status.DATA_IS_NULL, "fileContent"); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java index 11b633c854..725ffc00b6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.api.service; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.fasterxml.jackson.core.type.TypeReference; import org.apache.dolphinscheduler.api.dto.gantt.GanttDto; import org.apache.dolphinscheduler.api.dto.gantt.Task; import org.apache.dolphinscheduler.api.enums.Status; @@ -531,7 +532,11 @@ public class ProcessInstanceService extends BaseDAGService { List globalParams = new ArrayList<>(); if (userDefinedParams != null && userDefinedParams.length() > 0) { - globalParams = JSONUtils.toList(userDefinedParams, Property.class); + try { + globalParams = JSONUtils.getMapper().readValue(userDefinedParams, new TypeReference>() {}); + } catch (Exception e) { + logger.error("parse list exception!", e); + } } @@ -540,7 +545,11 @@ public class ProcessInstanceService extends BaseDAGService { // global param string String globalParamStr = JSONUtils.toJsonString(globalParams); globalParamStr = ParameterUtils.convertParameterPlaceholders(globalParamStr, timeParams); - globalParams = JSONUtils.toList(globalParamStr, Property.class); + try { + globalParams = JSONUtils.getMapper().readValue(globalParamStr, new TypeReference>() {}); + } catch (Exception e) { + logger.error("parse list exception!", e); + } for (Property property : globalParams) { timeParams.put(property.getProp(), property.getValue()); } @@ -553,7 +562,13 @@ public class ProcessInstanceService extends BaseDAGService { String localParams = map.get(LOCAL_PARAMS); if (localParams != null && !localParams.isEmpty()) { localParams = ParameterUtils.convertParameterPlaceholders(localParams, timeParams); - List localParamsList = JSONUtils.toList(localParams, Property.class); + List localParamsList = new ArrayList<>(); + try { + localParamsList = JSONUtils.getMapper().readValue(localParams, new TypeReference>() {}); + } catch (Exception e) { + logger.error("parse list exception!", e); + } + Map localParamsMap = new HashMap<>(); localParamsMap.put("taskType",taskNode.getType()); localParamsMap.put("localParamsList",localParamsList); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index 7e3b119502..97212154a3 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -50,6 +50,11 @@ public class JSONUtils { objectMapper.configure(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true).setTimeZone(TimeZone.getDefault()); } + public static ObjectMapper getMapper() { + return objectMapper; + } + + public static ArrayNode createArrayNode() { return objectMapper.createArrayNode(); } @@ -124,7 +129,7 @@ public class JSONUtils { return objectMapper.readValue(json, new TypeReference>() { }); } catch (Exception e) { - logger.error("JSONArray.parseArray exception!", e); + logger.error("parse list exception!", e); } return new ArrayList<>(); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java index ae49f15852..23e90f98d4 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java @@ -21,15 +21,21 @@ import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.core.toolkit.StringUtils; +import com.fasterxml.jackson.core.type.TypeReference; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.utils.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import java.util.stream.Stream; /** @@ -37,10 +43,12 @@ import java.util.stream.Collectors; */ @TableName("t_ds_process_definition") public class ProcessDefinition { + private static final Logger logger = LoggerFactory.getLogger(ProcessDefinition.class); + /** * id */ - @TableId(value="id", type=IdType.AUTO) + @TableId(value = "id", type = IdType.AUTO) private int id; /** @@ -81,14 +89,14 @@ public class ProcessDefinition { /** * user defined parameter list */ - @TableField(exist=false) + @TableField(exist = false) private List globalParamList; /** * user define parameter map */ - @TableField(exist=false) - private Map globalParamMap; + @TableField(exist = false) + private Map globalParamMap; /** * create time @@ -145,7 +153,7 @@ public class ProcessDefinition { /** * schedule release state : online/offline */ - @TableField(exist=false) + @TableField(exist = false) private ReleaseState scheduleReleaseState; /** @@ -271,7 +279,11 @@ public class ProcessDefinition { } public void setGlobalParams(String globalParams) { - this.globalParamList = JSONUtils.toList(globalParams, Property.class); + try { + this.globalParamList = JSONUtils.getMapper().readValue(globalParams, new TypeReference>() {}); + } catch (IOException e) { + logger.error("json parse exception!", e); + } this.globalParams = globalParams; } @@ -285,10 +297,16 @@ public class ProcessDefinition { } public Map getGlobalParamMap() { - List propList; + List propList = new ArrayList<> (); if (globalParamMap == null && StringUtils.isNotEmpty(globalParams)) { - propList = JSONUtils.toList(globalParams, Property.class); + try { + propList = JSONUtils.getMapper().readValue(globalParams, new TypeReference>() { + }); + } catch (IOException e) { + logger.error("json parse exception!", e); + } + globalParamMap = propList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java new file mode 100644 index 0000000000..c8c72a76ee --- /dev/null +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.dao.entity; + +import org.junit.Assert; +import org.junit.Test; + +public class ProcessDefinitionTest { + + /** + * task instance sub process + */ + @Test + public void getGlobalParamMapTest() { + ProcessDefinition taskInstance = new ProcessDefinition(); + + //sub process + taskInstance.setGlobalParams("[{\"prop\":\"selenium_global_parameters_1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"selenium_global_parameters_value_1\"}]"); + + Assert.assertEquals(taskInstance.getGlobalParamMap().toString(),"{selenium_global_parameters_1=selenium_global_parameters_value_1}"); + + + + } +} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java index 4c5cdc86ea..17b4b17715 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.server.worker.runner; +import com.fasterxml.jackson.core.type.TypeReference; import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.Constants; @@ -152,7 +153,13 @@ public class TaskExecuteThread implements Runnable { // global params string String globalParamsStr = taskExecutionContext.getGlobalParams(); if (globalParamsStr != null) { - List globalParamsList = JSONUtils.toList(globalParamsStr, Property.class); + List globalParamsList = new ArrayList<>(); + + try { + globalParamsList = JSONUtils.getMapper().readValue(globalParamsStr, new TypeReference>() {}); + } catch (Exception e) { + logger.error("parse list exception!", e); + } globalParamsMap.putAll(globalParamsList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue))); } return globalParamsMap; diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index e8376c8b96..ea2fbe51fc 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.service.process; import com.cronutils.model.Cron; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.lang.ArrayUtils; import org.apache.dolphinscheduler.common.Constants; @@ -36,6 +37,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; +import java.io.IOException; import java.util.*; import java.util.stream.Collectors; @@ -751,8 +753,16 @@ public class ProcessService { * @return global params join */ private String joinGlobalParams(String parentGlobalParams, String subGlobalParams){ - List parentPropertyList = JSONUtils.toList(parentGlobalParams, Property.class); - List subPropertyList = JSONUtils.toList(subGlobalParams, Property.class); + List parentPropertyList = new ArrayList<>(); + List subPropertyList = new ArrayList<>(); + + try { + parentPropertyList = JSONUtils.getMapper().readValue(parentGlobalParams, new TypeReference>() {}); + subPropertyList = JSONUtils.getMapper().readValue(subGlobalParams, new TypeReference>() {}); + } catch (IOException e) { + logger.error("json parse exception!", e); + } + Map subMap = subPropertyList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); for(Property parent : parentPropertyList){ From 4197ee8a8f3fcf238cba3186db04bf5f18ac2073 Mon Sep 17 00:00:00 2001 From: simon824 Date: Thu, 4 Jun 2020 16:46:11 +0800 Subject: [PATCH 15/38] Using Jackson instead of Fastjson --- .../dao/mapper/ProcessDefinitionMapperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java index 9dafbe138c..5747d997c5 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java @@ -69,6 +69,8 @@ public class ProcessDefinitionMapperTest { processDefinition.setUserId(101); processDefinition.setUpdateTime(new Date()); processDefinition.setCreateTime(new Date()); + processDefinition.setGlobalParams("[{\"prop\":\"selenium_global_parameters_1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"selenium_global_parameters_value_1\"}]"); + processDefinitionMapper.insert(processDefinition); return processDefinition; } From 32863755eccabec6d88028011e0c394417b248fb Mon Sep 17 00:00:00 2001 From: simon824 Date: Thu, 4 Jun 2020 16:56:31 +0800 Subject: [PATCH 16/38] Using Jackson instead of Fastjson --- .../dao/mapper/ProcessDefinitionMapperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java index 5747d997c5..c838ef9ce4 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java @@ -145,6 +145,8 @@ public class ProcessDefinitionMapperTest { processDefinition.setCreateTime(new Date()); processDefinition.setTenantId(tenant.getId()); processDefinition.setUserId(user.getId()); + processDefinition.setGlobalParams("[{\"prop\":\"selenium_global_parameters_1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"selenium_global_parameters_value_1\"}]"); + processDefinitionMapper.insert(processDefinition); ProcessDefinition processDefinition1 = processDefinitionMapper.queryByDefineName(project.getId(), "def 1"); From 9ada3c7aacbae76d23d3d49c68ce03b3c8336db1 Mon Sep 17 00:00:00 2001 From: simon824 Date: Thu, 4 Jun 2020 17:20:24 +0800 Subject: [PATCH 17/38] Using Jackson instead of Fastjson --- .../dao/entity/ProcessDefinition.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java index 23e90f98d4..2eb65b1263 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java @@ -279,10 +279,15 @@ public class ProcessDefinition { } public void setGlobalParams(String globalParams) { - try { - this.globalParamList = JSONUtils.getMapper().readValue(globalParams, new TypeReference>() {}); - } catch (IOException e) { - logger.error("json parse exception!", e); + if (globalParams == null){ + this.globalParamList = new ArrayList<>(); + }else { + try { + this.globalParamList = JSONUtils.getMapper().readValue(globalParams, new TypeReference>() { + }); + } catch (IOException e) { + logger.error("json parse exception!", e); + } } this.globalParams = globalParams; } From 9a92ab79822c1804fe5931ed3c2ec8e2dcb45eba Mon Sep 17 00:00:00 2001 From: simon824 Date: Thu, 4 Jun 2020 17:56:59 +0800 Subject: [PATCH 18/38] Using Jackson instead of Fastjson --- .../dao/mapper/ProcessDefinitionMapperTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java index c838ef9ce4..ec995df51c 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java @@ -69,7 +69,7 @@ public class ProcessDefinitionMapperTest { processDefinition.setUserId(101); processDefinition.setUpdateTime(new Date()); processDefinition.setCreateTime(new Date()); - processDefinition.setGlobalParams("[{\"prop\":\"selenium_global_parameters_1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"selenium_global_parameters_value_1\"}]"); +// processDefinition.setGlobalParams("[{\"prop\":\"selenium_global_parameters_1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"selenium_global_parameters_value_1\"}]"); processDefinitionMapper.insert(processDefinition); return processDefinition; @@ -145,8 +145,8 @@ public class ProcessDefinitionMapperTest { processDefinition.setCreateTime(new Date()); processDefinition.setTenantId(tenant.getId()); processDefinition.setUserId(user.getId()); - processDefinition.setGlobalParams("[{\"prop\":\"selenium_global_parameters_1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"selenium_global_parameters_value_1\"}]"); - +// processDefinition.setGlobalParams("[{\"prop\":\"selenium_global_parameters_1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"selenium_global_parameters_value_1\"}]"); +// processDefinitionMapper.insert(processDefinition); ProcessDefinition processDefinition1 = processDefinitionMapper.queryByDefineName(project.getId(), "def 1"); From 6e907f150c3b06a6fc11e0eee8ca0f403d101962 Mon Sep 17 00:00:00 2001 From: simon824 Date: Tue, 9 Jun 2020 13:02:50 +0800 Subject: [PATCH 19/38] 1. specified jackson Date format. 2. remove getMapper method,replace with toList() 3. rewrite toList method,support constructCollectionType. 4. parseObject(resp, Map.class) replace to toMap() method 5. rename JacksonSerialize to JsonSerializer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rewrite createArrayNode,createObjectNode method will Affect too many lines ，i will fix it in the next pr. --- .../template/impl/DefaultHTMLTemplate.java | 15 ++-- .../alert/utils/EnterpriseWeChatUtils.java | 72 +++++++++++-------- .../impl/DefaultHTMLTemplateTest.java | 2 +- .../api/dto/ScheduleParam.java | 4 ++ .../dolphinscheduler/api/dto/gantt/Task.java | 5 ++ .../api/dto/treeview/Instance.java | 4 ++ .../api/service/DataSourceService.java | 6 +- .../api/service/ProcessDefinitionService.java | 7 +- .../api/service/ProcessInstanceService.java | 18 +---- .../service/ProcessDefinitionServiceTest.java | 1 - .../common/model/DateInterval.java | 6 +- .../dolphinscheduler/common/model/Server.java | 4 ++ .../common/utils/HadoopUtils.java | 9 ++- .../common/utils/JSONUtils.java | 16 ++--- .../dao/entity/AlertGroup.java | 1 + .../dao/entity/CycleDependency.java | 3 + .../dao/entity/DataSource.java | 3 + .../dao/entity/ErrorCommand.java | 4 ++ .../dao/entity/MonitorRecord.java | 2 + .../dao/entity/ProcessDefinition.java | 18 ++--- .../dao/entity/ProcessInstance.java | 5 ++ .../dolphinscheduler/dao/entity/Queue.java | 3 + .../dolphinscheduler/dao/entity/Resource.java | 3 + .../dao/entity/ResourcesUser.java | 3 + .../dolphinscheduler/dao/entity/Schedule.java | 5 ++ .../dolphinscheduler/dao/entity/Session.java | 2 + .../dao/entity/TaskInstance.java | 4 ++ .../dao/entity/TaskRecord.java | 4 ++ .../dolphinscheduler/dao/entity/Tenant.java | 3 + .../dolphinscheduler/dao/entity/UDFUser.java | 3 + .../dolphinscheduler/dao/entity/UdfFunc.java | 3 + .../dolphinscheduler/dao/entity/User.java | 3 + .../dao/entity/UserAlertGroup.java | 3 + .../dao/entity/WorkerGroup.java | 4 +- .../dao/entity/WorkerServer.java | 4 ++ .../dao/entity/ProcessDefinitionTest.java | 1 + .../remote/command/TaskExecuteAckCommand.java | 2 +- .../command/TaskExecuteRequestCommand.java | 2 +- .../command/TaskExecuteResponseCommand.java | 2 +- .../remote/command/TaskInfo.java | 4 ++ .../command/TaskKillRequestCommand.java | 2 +- .../command/TaskKillResponseCommand.java | 2 +- .../log/GetLogBytesRequestCommand.java | 4 +- .../log/GetLogBytesResponseCommand.java | 4 +- .../log/RollViewLogRequestCommand.java | 4 +- .../log/RollViewLogResponseCommand.java | 4 +- .../command/log/ViewLogRequestCommand.java | 4 +- .../command/log/ViewLogResponseCommand.java | 4 +- ...sonSerializer.java => JsonSerializer.java} | 4 +- ...lizerTest.java => JsonSerializerTest.java} | 8 +-- .../server/entity/TaskExecutionContext.java | 7 +- .../server/log/LoggerRequestProcessor.java | 8 +-- .../master/processor/TaskAckProcessor.java | 4 +- .../processor/TaskKillResponseProcessor.java | 4 +- .../processor/TaskResponseProcessor.java | 4 +- .../processor/queue/TaskResponseEvent.java | 3 + .../runner/DependentTaskExecThread.java | 2 + .../processor/TaskExecuteProcessor.java | 4 +- .../worker/processor/TaskKillProcessor.java | 4 +- .../worker/runner/TaskExecuteThread.java | 7 +- .../server/worker/task/TaskProps.java | 3 + .../server/worker/task/datax/DataxTask.java | 7 +- .../worker/task/datax/DataxTaskTest.java | 1 + .../service/log/LogClientService.java | 8 +-- .../service/process/ProcessService.java | 10 +-- pom.xml | 2 +- 66 files changed, 228 insertions(+), 153 deletions(-) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/{JacksonSerializer.java => JsonSerializer.java} (95%) rename dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/{JacksonSerializerTest.java => JsonSerializerTest.java} (84%) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java index 063a4a4e89..7331df3913 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java @@ -16,6 +16,8 @@ */ package org.apache.dolphinscheduler.alert.template.impl; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import org.apache.dolphinscheduler.alert.template.AlertTemplate; import org.apache.dolphinscheduler.alert.utils.Constants; import org.apache.dolphinscheduler.common.enums.ShowType; @@ -107,18 +109,11 @@ public class DefaultHTMLTemplate implements AlertTemplate { private String getTextTypeMessage(String content,boolean showAll){ if (StringUtils.isNotEmpty(content)){ - List list; - try { - list = JSONUtils.toList(content,String.class); - }catch (Exception e){ - logger.error("json format exception",e); - return null; - } - + ArrayNode list = JSONUtils.parseArray(content); StringBuilder contents = new StringBuilder(100); - for (Object obj : list){ + for (JsonNode jsonNode : list){ contents.append(Constants.TR); - contents.append(Constants.TD).append(obj).append(Constants.TD_END); + contents.append(Constants.TD).append(jsonNode.toString()).append(Constants.TD_END); contents.append(Constants.TR_END); } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index 03cd53c0cd..bebb70bae7 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -64,14 +64,15 @@ public class EnterpriseWeChatUtils { /** * get Enterprise WeChat is enable + * * @return isEnable */ - public static boolean isEnable(){ + public static boolean isEnable() { Boolean isEnable = null; try { isEnable = PropertyUtils.getBoolean(Constants.ENTERPRISE_WECHAT_ENABLE); } catch (Exception e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); } if (isEnable == null) { return false; @@ -81,6 +82,7 @@ public class EnterpriseWeChatUtils { /** * get Enterprise WeChat token info + * * @return token string info * @throws IOException the IOException */ @@ -99,8 +101,12 @@ public class EnterpriseWeChatUtils { response.close(); } - Map map = JSONUtils.parseObject(resp, Map.class); - return map.get("access_token").toString(); + Map map = JSONUtils.toMap(resp); + if (map != null) { + return map.get("access_token"); + } else { + return null; + } } finally { httpClient.close(); } @@ -108,9 +114,10 @@ public class EnterpriseWeChatUtils { /** * make team single Enterprise WeChat message + * * @param toParty the toParty * @param agentId the agentId - * @param msg the msg + * @param msg the msg * @return Enterprise WeChat send message */ public static String makeTeamSendMsg(String toParty, String agentId, String msg) { @@ -121,9 +128,10 @@ public class EnterpriseWeChatUtils { /** * make team multi Enterprise WeChat message + * * @param toParty the toParty * @param agentId the agentId - * @param msg the msg + * @param msg the msg * @return Enterprise WeChat send message */ public static String makeTeamSendMsg(Collection toParty, String agentId, String msg) { @@ -135,9 +143,10 @@ public class EnterpriseWeChatUtils { /** * make team single user message - * @param toUser the toUser + * + * @param toUser the toUser * @param agentId the agentId - * @param msg the msg + * @param msg the msg * @return Enterprise WeChat send message */ public static String makeUserSendMsg(String toUser, String agentId, String msg) { @@ -148,9 +157,10 @@ public class EnterpriseWeChatUtils { /** * make team multi user message - * @param toUser the toUser + * + * @param toUser the toUser * @param agentId the agentId - * @param msg the msg + * @param msg the msg * @return Enterprise WeChat send message */ public static String makeUserSendMsg(Collection toUser, String agentId, String msg) { @@ -162,9 +172,10 @@ public class EnterpriseWeChatUtils { /** * send Enterprise WeChat + * * @param charset the charset - * @param data the data - * @param token the token + * @param data the data + * @param token the token * @return Enterprise WeChat resp, demo: {"errcode":0,"errmsg":"ok","invaliduser":""} * @throws IOException the IOException */ @@ -194,21 +205,22 @@ public class EnterpriseWeChatUtils { /** * convert table to markdown style - * @param title the title + * + * @param title the title * @param content the content * @return markdown table content */ - public static String markdownTable(String title,String content){ + public static String markdownTable(String title, String content) { List mapItemsList = JSONUtils.toList(content, LinkedHashMap.class); StringBuilder contents = new StringBuilder(200); if (null != mapItemsList) { - for (LinkedHashMap mapItems : mapItemsList){ + for (LinkedHashMap mapItems : mapItemsList) { Set> entries = mapItems.entrySet(); Iterator> iterator = entries.iterator(); - StringBuilder t = new StringBuilder(String.format("`%s`%s",title,Constants.MARKDOWN_ENTER)); + StringBuilder t = new StringBuilder(String.format("`%s`%s", title, Constants.MARKDOWN_ENTER)); - while (iterator.hasNext()){ + while (iterator.hasNext()) { Map.Entry entry = iterator.next(); t.append(Constants.MARKDOWN_QUOTE); @@ -223,23 +235,24 @@ public class EnterpriseWeChatUtils { /** * convert text to markdown style - * @param title the title + * + * @param title the title * @param content the content * @return markdown text */ - public static String markdownText(String title,String content){ - if (StringUtils.isNotEmpty(content)){ + public static String markdownText(String title, String content) { + if (StringUtils.isNotEmpty(content)) { List list; try { - list = JSONUtils.toList(content,String.class); - }catch (Exception e){ - logger.error("json format exception",e); + list = JSONUtils.toList(content, String.class); + } catch (Exception e) { + logger.error("json format exception", e); return null; } StringBuilder contents = new StringBuilder(100); - contents.append(String.format("`%s`%n",title)); - for (String str : list){ + contents.append(String.format("`%s`%n", title)); + for (String str : list) { contents.append(Constants.MARKDOWN_QUOTE); contents.append(str); contents.append(Constants.MARKDOWN_ENTER); @@ -253,14 +266,15 @@ public class EnterpriseWeChatUtils { /** * Determine the mardown style based on the show type of the alert + * * @return the markdown alert table/text */ - public static String markdownByAlert(AlertData alert){ + public static String markdownByAlert(AlertData alert) { String result = ""; if (alert.getShowType().equals(ShowType.TABLE.getDescp())) { - result = markdownTable(alert.getTitle(),alert.getContent()); - }else if(alert.getShowType().equals(ShowType.TEXT.getDescp())){ - result = markdownText(alert.getTitle(),alert.getContent()); + result = markdownTable(alert.getTitle(), alert.getContent()); + } else if (alert.getShowType().equals(ShowType.TEXT.getDescp())) { + result = markdownText(alert.getTitle(), alert.getContent()); } return result; diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplateTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplateTest.java index 3de6724246..76d01c134d 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplateTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplateTest.java @@ -116,7 +116,7 @@ public class DefaultHTMLTemplateTest{ " \n" + " \n" + " \n" + - "
{mysql service name=mysql200, mysql address=192.168.xx.xx, port=3306, no index of number=80, database client connections=190}
{mysql service name=mysql210, mysql address=192.168.xx.xx, port=3306, no index of number=10, database client connections=90}
\n" + + "{\"mysql service name\":\"mysql200\",\"mysql address\":\"192.168.xx.xx\",\"port\":\"3306\",\"no index of number\":\"80\",\"database client connections\":\"190\"}{\"mysql service name\":\"mysql210\",\"mysql address\":\"192.168.xx.xx\",\"port\":\"3306\",\"no index of number\":\"10\",\"database client connections\":\"90\"} \n" + " \n" + ""; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/ScheduleParam.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/ScheduleParam.java index a842960bc2..a7cdfcfe46 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/ScheduleParam.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/ScheduleParam.java @@ -16,13 +16,17 @@ */ package org.apache.dolphinscheduler.api.dto; +import com.fasterxml.jackson.annotation.JsonFormat; + import java.util.Date; /** * schedule parameters */ public class ScheduleParam { + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date endTime; private String crontab; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/Task.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/Task.java index 55ead2872a..453559e123 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/Task.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/Task.java @@ -16,6 +16,8 @@ */ package org.apache.dolphinscheduler.api.dto.gantt; +import com.fasterxml.jackson.annotation.JsonFormat; + import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -41,16 +43,19 @@ public class Task { /** * task execution date */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date executionDate; /** * task iso start */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date isoStart; /** * task iso end */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date isoEnd; /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/treeview/Instance.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/treeview/Instance.java index 10bfeb0790..9bab520ed5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/treeview/Instance.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/treeview/Instance.java @@ -16,6 +16,8 @@ */ package org.apache.dolphinscheduler.api.dto.treeview; +import com.fasterxml.jackson.annotation.JsonFormat; + import java.util.Date; /** @@ -42,11 +44,13 @@ public class Instance { /** * node start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** * node end time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date endTime; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index a702ffe1e2..735cd8ce89 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -167,7 +167,7 @@ public class DataSourceService extends BaseService{ paramObject.put(Constants.PASSWORD, oldParams.path(Constants.PASSWORD).asText()); } // connectionParams json - String connectionParams = JSONUtils.toJsonString(paramObject); + String connectionParams = paramObject.toString(); Boolean isConnection = checkConnection(type, connectionParams); if (!isConnection) { @@ -315,7 +315,7 @@ public class DataSourceService extends BaseService{ String connectionParams = dataSource.getConnectionParams(); ObjectNode object = JSONUtils.parseObject(connectionParams); object.put(Constants.PASSWORD, Constants.XXXXXX); - dataSource.setConnectionParams(JSONUtils.toJsonString(object)); + dataSource.setConnectionParams(object.toString()); } } @@ -524,7 +524,7 @@ public class DataSourceService extends BaseService{ parameterMap.put(Constants.PRINCIPAL,principal); } if (other != null && !"".equals(other)) { - LinkedHashMap map = JSONUtils.parseObject(other, LinkedHashMap.class); + Map map = JSONUtils.toMap(other); if (map.size() > 0) { StringBuilder otherSb = new StringBuilder(); for (Map.Entry entry: map.entrySet()) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index b76bfd00e3..c3d8516ed0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -756,13 +756,8 @@ public class ProcessDefinitionService extends BaseDAGService { public Map importProcessDefinition(User loginUser, MultipartFile file, String currentProjectName) { Map result = new HashMap<>(5); String processMetaJson = FileUtils.file2String(file); - List processMetaList = new ArrayList<>(); + List processMetaList = JSONUtils.toList(processMetaJson, ProcessMeta.class); - try { - processMetaList = JSONUtils.getMapper().readValue(processMetaJson, new TypeReference>() {}); - } catch (Exception e) { - logger.error("parse list exception!", e); - } //check file content if (CollectionUtils.isEmpty(processMetaList)) { putMsg(result, Status.DATA_IS_NULL, "fileContent"); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java index 725ffc00b6..5d5eb006f8 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java @@ -532,11 +532,7 @@ public class ProcessInstanceService extends BaseDAGService { List globalParams = new ArrayList<>(); if (userDefinedParams != null && userDefinedParams.length() > 0) { - try { - globalParams = JSONUtils.getMapper().readValue(userDefinedParams, new TypeReference>() {}); - } catch (Exception e) { - logger.error("parse list exception!", e); - } + globalParams = JSONUtils.toList(userDefinedParams, Property.class); } @@ -545,11 +541,7 @@ public class ProcessInstanceService extends BaseDAGService { // global param string String globalParamStr = JSONUtils.toJsonString(globalParams); globalParamStr = ParameterUtils.convertParameterPlaceholders(globalParamStr, timeParams); - try { - globalParams = JSONUtils.getMapper().readValue(globalParamStr, new TypeReference>() {}); - } catch (Exception e) { - logger.error("parse list exception!", e); - } + globalParams = JSONUtils.toList(globalParamStr, Property.class); for (Property property : globalParams) { timeParams.put(property.getProp(), property.getValue()); } @@ -563,11 +555,7 @@ public class ProcessInstanceService extends BaseDAGService { if (localParams != null && !localParams.isEmpty()) { localParams = ParameterUtils.convertParameterPlaceholders(localParams, timeParams); List localParamsList = new ArrayList<>(); - try { - localParamsList = JSONUtils.getMapper().readValue(localParams, new TypeReference>() {}); - } catch (Exception e) { - logger.error("parse list exception!", e); - } + localParamsList = JSONUtils.toList(localParams, Property.class); Map localParamsMap = new HashMap<>(); localParamsMap.put("taskType",taskNode.getType()); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index 618f25ac2a..d4fa2307ec 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -521,7 +521,6 @@ public class ProcessDefinitionServiceTest { @Test public void testExportProcessMetaDataStr() { Mockito.when(scheduleMapper.queryByProcessDefinitionId(46)).thenReturn(getSchedulerList()); - ProcessDefinition processDefinition = getProcessDefinition(); processDefinition.setProcessDefinitionJson(sqlDependentJson); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/DateInterval.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/DateInterval.java index e936a444d0..2ff3922c18 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/DateInterval.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/DateInterval.java @@ -16,6 +16,8 @@ */ package org.apache.dolphinscheduler.common.model; +import com.fasterxml.jackson.annotation.JsonFormat; + import java.util.Date; import java.util.Objects; @@ -23,9 +25,9 @@ import java.util.Objects; * date interval class */ public class DateInterval { - + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; - + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date endTime; public DateInterval(Date beginTime, Date endTime){ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/Server.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/Server.java index 706609df56..69d718c286 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/Server.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/Server.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.common.model; +import com.fasterxml.jackson.annotation.JsonFormat; + import java.util.Date; /** @@ -52,11 +54,13 @@ public class Server { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * laster heart beat time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date lastHeartbeatTime; public int getId() { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 360f4514b6..e57201d74d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; +import com.fasterxml.jackson.core.io.JsonEOFException; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -407,7 +408,7 @@ public class HadoopUtils implements Closeable { * @param applicationId application id * @return the return may be null or there may be other parse exceptions */ - public ExecutionStatus getApplicationStatus(String applicationId) { + public ExecutionStatus getApplicationStatus(String applicationId) throws NullPointerException { if (StringUtils.isEmpty(applicationId)) { return null; } @@ -426,6 +427,9 @@ public class HadoopUtils implements Closeable { logger.info("jobHistoryUrl={}", jobHistoryUrl); responseContent = HttpUtils.get(jobHistoryUrl); ObjectNode jsonObject = JSONUtils.parseObject(responseContent); + if (!jsonObject.has("job")){ + throw new NullPointerException(); + } result = jsonObject.path("job").path("state").asText(); } @@ -670,6 +674,9 @@ public class HadoopUtils implements Closeable { ObjectNode jsonObject = JSONUtils.parseObject(retStr); //get ResourceManager state + if (!jsonObject.has("clusterInfo")){ + return null; + } return jsonObject.get("clusterInfo").path("haState").asText(); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index 97212154a3..63611cc7d6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.*; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.type.CollectionType; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,10 +51,6 @@ public class JSONUtils { objectMapper.configure(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true).setTimeZone(TimeZone.getDefault()); } - public static ObjectMapper getMapper() { - return objectMapper; - } - public static ArrayNode createArrayNode() { return objectMapper.createArrayNode(); @@ -63,7 +60,6 @@ public class JSONUtils { return objectMapper.createObjectNode(); } - public static JsonNode toJsonNode(Object obj) { return objectMapper.valueToTree(obj); } @@ -125,9 +121,11 @@ public class JSONUtils { if (StringUtils.isEmpty(json)) { return new ArrayList<>(); } + try { - return objectMapper.readValue(json, new TypeReference>() { - }); + + CollectionType listType = objectMapper.getTypeFactory().constructCollectionType(ArrayList.class, clazz); + return objectMapper.readValue(json, listType); } catch (Exception e) { logger.error("parse list exception!", e); } @@ -189,7 +187,7 @@ public class JSONUtils { */ public static Map toMap(String json) { if (StringUtils.isEmpty(json)) { - return null; + return new HashMap<>(); } try { @@ -198,7 +196,7 @@ public class JSONUtils { logger.error("json to map exception!", e); } - return null; + return new HashMap<>(); } /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AlertGroup.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AlertGroup.java index e35e2f759c..652d63bc70 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AlertGroup.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AlertGroup.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.dao.entity; import com.baomidou.mybatisplus.annotation.TableField; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.AlertType; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CycleDependency.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CycleDependency.java index e7ad38b6ea..89a57aae95 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CycleDependency.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CycleDependency.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.dao.entity; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.CycleEnum; import java.util.Date; @@ -31,10 +32,12 @@ public class CycleDependency { /** * last schedule time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date lastScheduleTime; /** * expiration time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date expirationTime; /** * cycle enum diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java index 236ae8f91e..ecd5be0455 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.dao.entity; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.DbType; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; @@ -66,11 +67,13 @@ public class DataSource { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; public DataSource() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java index 127c5b7322..760bb23d90 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.dao.entity; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.*; import java.util.Date; @@ -78,11 +79,13 @@ public class ErrorCommand { /** * schedule time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date scheduleTime; /** * start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** @@ -93,6 +96,7 @@ public class ErrorCommand { /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/MonitorRecord.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/MonitorRecord.java index 259827ed80..318f24535f 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/MonitorRecord.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/MonitorRecord.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.dao.entity; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.Flag; @@ -56,6 +57,7 @@ public class MonitorRecord { /** * start date */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date date; public Flag getState() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java index 2eb65b1263..6b4a83dfd8 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java @@ -21,6 +21,7 @@ import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.core.toolkit.StringUtils; +import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.core.type.TypeReference; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.ReleaseState; @@ -101,11 +102,13 @@ public class ProcessDefinition { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; /** @@ -282,12 +285,7 @@ public class ProcessDefinition { if (globalParams == null){ this.globalParamList = new ArrayList<>(); }else { - try { - this.globalParamList = JSONUtils.getMapper().readValue(globalParams, new TypeReference>() { - }); - } catch (IOException e) { - logger.error("json parse exception!", e); - } + this.globalParamList = JSONUtils.toList(globalParams, Property.class); } this.globalParams = globalParams; } @@ -305,13 +303,7 @@ public class ProcessDefinition { List propList = new ArrayList<> (); if (globalParamMap == null && StringUtils.isNotEmpty(globalParams)) { - try { - propList = JSONUtils.getMapper().readValue(globalParams, new TypeReference>() { - }); - } catch (IOException e) { - logger.error("json parse exception!", e); - } - + propList = JSONUtils.toList(globalParams,Property.class); globalParamMap = propList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java index 2fa8e64451..28f1eba975 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java @@ -21,6 +21,7 @@ import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.core.toolkit.StringUtils; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.*; import java.util.Date; @@ -52,11 +53,13 @@ public class ProcessInstance { /** * start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** * end time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date endTime; /** @@ -117,11 +120,13 @@ public class ProcessInstance { /** * schedule time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date scheduleTime; /** * command start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date commandStartTime; /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Queue.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Queue.java index 57b678d2a7..9c89167d76 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Queue.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Queue.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.dao.entity; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; import java.util.Date; @@ -45,10 +46,12 @@ public class Queue { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; public int getId() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java index 16d94914fd..214fbe83e4 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.dao.entity; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.ResourceType; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; @@ -80,11 +81,13 @@ public class Resource { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; public Resource() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesUser.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesUser.java index dc62a7b66f..ea8bd6c1b9 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesUser.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesUser.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.dao.entity; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; import java.util.Date; @@ -52,11 +53,13 @@ public class ResourcesUser { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; public int getId() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java index 0cb41080b2..5d2bc3811e 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java @@ -20,6 +20,7 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.FailureStrategy; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.ReleaseState; @@ -62,11 +63,13 @@ public class Schedule { /** * schedule start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** * schedule end time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date endTime; /** @@ -87,11 +90,13 @@ public class Schedule { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Session.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Session.java index 39bf6143dd..060d8824d3 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Session.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Session.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.dao.entity; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; import java.util.Date; @@ -42,6 +43,7 @@ public class Session { /** * last login time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date lastLoginTime; /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java index 561876d0b3..eb51fc50cb 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.dao.entity; import com.baomidou.mybatisplus.annotation.TableField; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Priority; @@ -84,16 +85,19 @@ public class TaskInstance implements Serializable { /** * task submit time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date submitTime; /** * task start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** * task end time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date endTime; /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskRecord.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskRecord.java index 9a6979525b..b267dff80c 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskRecord.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskRecord.java @@ -16,6 +16,8 @@ */ package org.apache.dolphinscheduler.dao.entity; +import com.fasterxml.jackson.annotation.JsonFormat; + import java.util.Date; /** @@ -46,11 +48,13 @@ public class TaskRecord { /** * start date */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** * end date */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date endTime; /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Tenant.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Tenant.java index a289f4f1f5..1b2eea403d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Tenant.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Tenant.java @@ -20,6 +20,7 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; import java.util.Date; import java.util.Objects; @@ -71,10 +72,12 @@ public class Tenant { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UDFUser.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UDFUser.java index c21eb96abd..fa3fbc2771 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UDFUser.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UDFUser.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.dao.entity; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; import java.util.Date; @@ -52,11 +53,13 @@ public class UDFUser { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; public int getId() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java index e14255be77..ef0badf7e5 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.dao.entity; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.UdfType; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; @@ -81,11 +82,13 @@ public class UdfFunc { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; public int getId() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java index b4f2f93dcc..89da171caf 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.dao.entity; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.UserType; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; @@ -104,11 +105,13 @@ public class User { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; public int getId() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UserAlertGroup.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UserAlertGroup.java index e86b32e075..902196d186 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UserAlertGroup.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UserAlertGroup.java @@ -20,6 +20,7 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; import java.util.Date; @@ -61,11 +62,13 @@ public class UserAlertGroup { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; public int getId() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerGroup.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerGroup.java index bce963686c..b850219836 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerGroup.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerGroup.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.dao.entity; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; import java.util.Date; import java.util.List; @@ -32,8 +33,9 @@ public class WorkerGroup { private List ipList; + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; - + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerServer.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerServer.java index 977302061b..58bf2d24d5 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerServer.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerServer.java @@ -16,6 +16,8 @@ */ package org.apache.dolphinscheduler.dao.entity; +import com.fasterxml.jackson.annotation.JsonFormat; + import java.util.Date; public class WorkerServer { @@ -49,11 +51,13 @@ public class WorkerServer { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * last heart beat time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date lastHeartbeatTime; public int getId() { diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java index c8c72a76ee..b342f01743 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java @@ -31,6 +31,7 @@ public class ProcessDefinitionTest { //sub process taskInstance.setGlobalParams("[{\"prop\":\"selenium_global_parameters_1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"selenium_global_parameters_value_1\"}]"); + taskInstance.getGlobalParamMap(); Assert.assertEquals(taskInstance.getGlobalParamMap().toString(),"{selenium_global_parameters_1=selenium_global_parameters_value_1}"); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteAckCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteAckCommand.java index a965849ca3..796e4dc358 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteAckCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteAckCommand.java @@ -1 +1 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import java.io.Serializable; import java.util.Date; /** * execute task request command */ public class TaskExecuteAckCommand implements Serializable { /** * taskInstanceId */ private int taskInstanceId; /** * startTime */ private Date startTime; /** * host */ private String host; /** * status */ private int status; /** * logPath */ private String logPath; /** * executePath */ private String executePath; public Date getStartTime() { return startTime; } public void setStartTime(Date startTime) { this.startTime = startTime; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } public String getLogPath() { return logPath; } public void setLogPath(String logPath) { this.logPath = logPath; } public String getExecutePath() { return executePath; } public void setExecutePath(String executePath) { this.executePath = executePath; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_EXECUTE_ACK); byte[] body = JacksonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskExecuteAckCommand{" + "taskInstanceId=" + taskInstanceId + ", startTime=" + startTime + ", host='" + host + '\'' + ", status=" + status + ", logPath='" + logPath + '\'' + ", executePath='" + executePath + '\'' + '}'; } } \ No newline at end of file +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; import java.util.Date; /** * execute task request command */ public class TaskExecuteAckCommand implements Serializable { /** * taskInstanceId */ private int taskInstanceId; /** * startTime */ @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** * host */ private String host; /** * status */ private int status; /** * logPath */ private String logPath; /** * executePath */ private String executePath; public Date getStartTime() { return startTime; } public void setStartTime(Date startTime) { this.startTime = startTime; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } public String getLogPath() { return logPath; } public void setLogPath(String logPath) { this.logPath = logPath; } public String getExecutePath() { return executePath; } public void setExecutePath(String executePath) { this.executePath = executePath; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_EXECUTE_ACK); byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskExecuteAckCommand{" + "taskInstanceId=" + taskInstanceId + ", startTime=" + startTime + ", host='" + host + '\'' + ", status=" + status + ", logPath='" + logPath + '\'' + ", executePath='" + executePath + '\'' + '}'; } } \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java index affa17c08f..603f8a088b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java @@ -1 +1 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import java.io.Serializable; /** * execute task request command */ public class TaskExecuteRequestCommand implements Serializable { /** * task execution context */ private String taskExecutionContext; public String getTaskExecutionContext() { return taskExecutionContext; } public void setTaskExecutionContext(String taskExecutionContext) { this.taskExecutionContext = taskExecutionContext; } public TaskExecuteRequestCommand() { } public TaskExecuteRequestCommand(String taskExecutionContext) { this.taskExecutionContext = taskExecutionContext; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_EXECUTE_REQUEST); byte[] body = JacksonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskExecuteRequestCommand{" + "taskExecutionContext='" + taskExecutionContext + '\'' + '}'; } } \ No newline at end of file +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; /** * execute task request command */ public class TaskExecuteRequestCommand implements Serializable { /** * task execution context */ private String taskExecutionContext; public String getTaskExecutionContext() { return taskExecutionContext; } public void setTaskExecutionContext(String taskExecutionContext) { this.taskExecutionContext = taskExecutionContext; } public TaskExecuteRequestCommand() { } public TaskExecuteRequestCommand(String taskExecutionContext) { this.taskExecutionContext = taskExecutionContext; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_EXECUTE_REQUEST); byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskExecuteRequestCommand{" + "taskExecutionContext='" + taskExecutionContext + '\'' + '}'; } } \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java index 4ac4bbb36a..0ea34fe39f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java @@ -1 +1 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import java.io.Serializable; import java.util.Date; /** * execute task response command */ public class TaskExecuteResponseCommand implements Serializable { public TaskExecuteResponseCommand() { } public TaskExecuteResponseCommand(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } /** * task instance id */ private int taskInstanceId; /** * status */ private int status; /** * end time */ private Date endTime; /** * processId */ private int processId; /** * appIds */ private String appIds; public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public Date getEndTime() { return endTime; } public void setEndTime(Date endTime) { this.endTime = endTime; } public int getProcessId() { return processId; } public void setProcessId(int processId) { this.processId = processId; } public String getAppIds() { return appIds; } public void setAppIds(String appIds) { this.appIds = appIds; } /** * package response command * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_EXECUTE_RESPONSE); byte[] body = JacksonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskExecuteResponseCommand{" + "taskInstanceId=" + taskInstanceId + ", status=" + status + ", endTime=" + endTime + ", processId=" + processId + ", appIds='" + appIds + '\'' + '}'; } } \ No newline at end of file +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; import java.util.Date; /** * execute task response command */ public class TaskExecuteResponseCommand implements Serializable { public TaskExecuteResponseCommand() { } public TaskExecuteResponseCommand(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } /** * task instance id */ private int taskInstanceId; /** * status */ private int status; /** * end time */ @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date endTime; /** * processId */ private int processId; /** * appIds */ private String appIds; public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public Date getEndTime() { return endTime; } public void setEndTime(Date endTime) { this.endTime = endTime; } public int getProcessId() { return processId; } public void setProcessId(int processId) { this.processId = processId; } public String getAppIds() { return appIds; } public void setAppIds(String appIds) { this.appIds = appIds; } /** * package response command * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_EXECUTE_RESPONSE); byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskExecuteResponseCommand{" + "taskInstanceId=" + taskInstanceId + ", status=" + status + ", endTime=" + endTime + ", processId=" + processId + ", appIds='" + appIds + '\'' + '}'; } } \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskInfo.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskInfo.java index 3fb58fe3da..196d0a73ae 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskInfo.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskInfo.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.remote.command; +import com.fasterxml.jackson.annotation.JsonFormat; + import java.io.Serializable; import java.util.Date; @@ -39,6 +41,7 @@ public class TaskInfo implements Serializable{ /** * task start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** @@ -66,6 +69,7 @@ public class TaskInfo implements Serializable{ /** * process instance schedule time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date scheduleTime; /** diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java index d1855e42eb..092e462a1f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java @@ -1 +1 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import java.io.Serializable; /** * kill task request command */ public class TaskKillRequestCommand implements Serializable { /** * task id */ private int taskInstanceId; public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_KILL_REQUEST); byte[] body = JacksonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskKillRequestCommand{" + "taskInstanceId=" + taskInstanceId + '}'; } } \ No newline at end of file +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; /** * kill task request command */ public class TaskKillRequestCommand implements Serializable { /** * task id */ private int taskInstanceId; public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_KILL_REQUEST); byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskKillRequestCommand{" + "taskInstanceId=" + taskInstanceId + '}'; } } \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java index 2f35e58925..9dc0c541d7 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java @@ -1 +1 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; import java.io.Serializable; import java.util.List; /** * kill task response command */ public class TaskKillResponseCommand implements Serializable { /** * taskInstanceId */ private int taskInstanceId; /** * host */ private String host; /** * status */ private int status; /** * processId */ private int processId; /** * other resource manager appId , for example : YARN etc */ protected List appIds; public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public int getProcessId() { return processId; } public void setProcessId(int processId) { this.processId = processId; } public List getAppIds() { return appIds; } public void setAppIds(List appIds) { this.appIds = appIds; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_KILL_RESPONSE); byte[] body = JacksonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskKillResponseCommand{" + "taskInstanceId=" + taskInstanceId + ", host='" + host + '\'' + ", status=" + status + ", processId=" + processId + ", appIds=" + appIds + '}'; } } \ No newline at end of file +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; import java.util.List; /** * kill task response command */ public class TaskKillResponseCommand implements Serializable { /** * taskInstanceId */ private int taskInstanceId; /** * host */ private String host; /** * status */ private int status; /** * processId */ private int processId; /** * other resource manager appId , for example : YARN etc */ protected List appIds; public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public int getProcessId() { return processId; } public void setProcessId(int processId) { this.processId = processId; } public List getAppIds() { return appIds; } public void setAppIds(List appIds) { this.appIds = appIds; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_KILL_RESPONSE); byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskKillResponseCommand{" + "taskInstanceId=" + taskInstanceId + ", host='" + host + '\'' + ", status=" + status + ", processId=" + processId + ", appIds=" + appIds + '}'; } } \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java index 4dc9af486f..e4b21e2f89 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; @@ -56,7 +56,7 @@ public class GetLogBytesRequestCommand implements Serializable { public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.GET_LOG_BYTES_REQUEST); - byte[] body = JacksonSerializer.serialize(this); + byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java index 32d1a5bea9..349ec03855 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; @@ -57,7 +57,7 @@ public class GetLogBytesResponseCommand implements Serializable { public Command convert2Command(long opaque){ Command command = new Command(opaque); command.setType(CommandType.GET_LOG_BYTES_RESPONSE); - byte[] body = JacksonSerializer.serialize(this); + byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java index eeccc5860b..00129c7e78 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; @@ -84,7 +84,7 @@ public class RollViewLogRequestCommand implements Serializable { public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.ROLL_VIEW_LOG_REQUEST); - byte[] body = JacksonSerializer.serialize(this); + byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java index dca2feed20..a4f4f86c9b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; @@ -57,7 +57,7 @@ public class RollViewLogResponseCommand implements Serializable { public Command convert2Command(long opaque){ Command command = new Command(opaque); command.setType(CommandType.ROLL_VIEW_LOG_RESPONSE); - byte[] body = JacksonSerializer.serialize(this); + byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java index 646d05c327..1d51653eac 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; @@ -56,7 +56,7 @@ public class ViewLogRequestCommand implements Serializable { public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.VIEW_WHOLE_LOG_REQUEST); - byte[] body = JacksonSerializer.serialize(this); + byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java index 970d1c07b8..6940104f71 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; @@ -57,7 +57,7 @@ public class ViewLogResponseCommand implements Serializable { public Command convert2Command(long opaque){ Command command = new Command(opaque); command.setType(CommandType.VIEW_WHOLE_LOG_RESPONSE); - byte[] body = JacksonSerializer.serialize(this); + byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JacksonSerializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java similarity index 95% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JacksonSerializer.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java index 774510fe36..a23dec4a75 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JacksonSerializer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java @@ -27,9 +27,9 @@ import java.nio.charset.StandardCharsets; /** * json serialize or deserialize */ -public class JacksonSerializer { +public class JsonSerializer { private static final ObjectMapper objectMapper = new ObjectMapper(); - private static final Logger logger = LoggerFactory.getLogger(JacksonSerializer.class); + private static final Logger logger = LoggerFactory.getLogger(JsonSerializer.class); /** * serialize to byte diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/JacksonSerializerTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/JsonSerializerTest.java similarity index 84% rename from dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/JacksonSerializerTest.java rename to dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/JsonSerializerTest.java index 28ef0f98e2..cb92db7f25 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/JacksonSerializerTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/JsonSerializerTest.java @@ -18,20 +18,20 @@ package org.apache.dolphinscheduler.remote; -import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.junit.Assert; import org.junit.Test; -public class JacksonSerializerTest { +public class JsonSerializerTest { @Test public void testSerialize(){ TestObj testObj = new TestObj(); testObj.setAge(12); - byte[] serializeByte = JacksonSerializer.serialize(testObj); + byte[] serializeByte = JsonSerializer.serialize(testObj); // - TestObj deserialize = JacksonSerializer.deserialize(serializeByte, TestObj.class); + TestObj deserialize = JsonSerializer.deserialize(serializeByte, TestObj.class); Assert.assertEquals(testObj.getAge(), deserialize.getAge()); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java index 148f04c649..7b4c721422 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java @@ -17,9 +17,10 @@ package org.apache.dolphinscheduler.server.entity; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand; -import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; import java.util.Date; @@ -45,6 +46,7 @@ public class TaskExecutionContext implements Serializable{ /** * task start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** @@ -91,6 +93,7 @@ public class TaskExecutionContext implements Serializable{ /** * process instance schedule time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date scheduleTime; /** @@ -431,7 +434,7 @@ public class TaskExecutionContext implements Serializable{ public Command toCommand(){ TaskExecuteRequestCommand requestCommand = new TaskExecuteRequestCommand(); - requestCommand.setTaskExecutionContext(JacksonSerializer.serializeToString(this)); + requestCommand.setTaskExecutionContext(JsonSerializer.serializeToString(this)); return requestCommand.convert2Command(); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java index 247597080a..b7eea01f31 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java @@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.log.*; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,21 +61,21 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { final CommandType commandType = command.getType(); switch (commandType){ case GET_LOG_BYTES_REQUEST: - GetLogBytesRequestCommand getLogRequest = JacksonSerializer.deserialize( + GetLogBytesRequestCommand getLogRequest = JsonSerializer.deserialize( command.getBody(), GetLogBytesRequestCommand.class); byte[] bytes = getFileContentBytes(getLogRequest.getPath()); GetLogBytesResponseCommand getLogResponse = new GetLogBytesResponseCommand(bytes); channel.writeAndFlush(getLogResponse.convert2Command(command.getOpaque())); break; case VIEW_WHOLE_LOG_REQUEST: - ViewLogRequestCommand viewLogRequest = JacksonSerializer.deserialize( + ViewLogRequestCommand viewLogRequest = JsonSerializer.deserialize( command.getBody(), ViewLogRequestCommand.class); String msg = readWholeFileContent(viewLogRequest.getPath()); ViewLogResponseCommand viewLogResponse = new ViewLogResponseCommand(msg); channel.writeAndFlush(viewLogResponse.convert2Command(command.getOpaque())); break; case ROLL_VIEW_LOG_REQUEST: - RollViewLogRequestCommand rollViewLogRequest = JacksonSerializer.deserialize( + RollViewLogRequestCommand rollViewLogRequest = JsonSerializer.deserialize( command.getBody(), RollViewLogRequestCommand.class); List lines = readPartFileContent(rollViewLogRequest.getPath(), rollViewLogRequest.getSkipLineNum(), rollViewLogRequest.getLimit()); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java index b695310cae..e579535440 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java @@ -28,7 +28,7 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.remote.utils.ChannelUtils; -import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.apache.dolphinscheduler.server.master.cache.TaskInstanceCacheManager; import org.apache.dolphinscheduler.server.master.cache.impl.TaskInstanceCacheManagerImpl; import org.apache.dolphinscheduler.server.master.processor.queue.TaskResponseEvent; @@ -77,7 +77,7 @@ public class TaskAckProcessor implements NettyRequestProcessor { @Override public void process(Channel channel, Command command) { Preconditions.checkArgument(CommandType.TASK_EXECUTE_ACK == command.getType(), String.format("invalid command type : %s", command.getType())); - TaskExecuteAckCommand taskAckCommand = JacksonSerializer.deserialize(command.getBody(), TaskExecuteAckCommand.class); + TaskExecuteAckCommand taskAckCommand = JsonSerializer.deserialize(command.getBody(), TaskExecuteAckCommand.class); logger.info("taskAckCommand : {}", taskAckCommand); taskInstanceCacheManager.cacheTaskInstance(taskAckCommand); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java index 720f8fd751..2e51998cbd 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java @@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskKillResponseCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +45,7 @@ public class TaskKillResponseProcessor implements NettyRequestProcessor { public void process(Channel channel, Command command) { Preconditions.checkArgument(CommandType.TASK_KILL_RESPONSE == command.getType(), String.format("invalid command type : %s", command.getType())); - TaskKillResponseCommand responseCommand = JacksonSerializer.deserialize(command.getBody(), TaskKillResponseCommand.class); + TaskKillResponseCommand responseCommand = JsonSerializer.deserialize(command.getBody(), TaskKillResponseCommand.class); logger.info("received task kill response command : {}", responseCommand); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java index 0d1214ca2b..b04b930fd4 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java @@ -27,7 +27,7 @@ import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.apache.dolphinscheduler.server.master.cache.TaskInstanceCacheManager; import org.apache.dolphinscheduler.server.master.cache.impl.TaskInstanceCacheManagerImpl; import org.apache.dolphinscheduler.server.master.processor.queue.TaskResponseEvent; @@ -78,7 +78,7 @@ public class TaskResponseProcessor implements NettyRequestProcessor { public void process(Channel channel, Command command) { Preconditions.checkArgument(CommandType.TASK_EXECUTE_RESPONSE == command.getType(), String.format("invalid command type : %s", command.getType())); - TaskExecuteResponseCommand responseCommand = JacksonSerializer.deserialize(command.getBody(), TaskExecuteResponseCommand.class); + TaskExecuteResponseCommand responseCommand = JsonSerializer.deserialize(command.getBody(), TaskExecuteResponseCommand.class); logger.info("received command : {}", responseCommand); taskInstanceCacheManager.cacheTaskInstance(responseCommand); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java index 9e8813fd7f..051cc388bf 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.master.processor.queue; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import java.util.Date; @@ -44,11 +45,13 @@ public class TaskResponseEvent { /** * start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** * end time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date endTime; /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java index 88ed29e55e..7c8da5ab8c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.server.master.runner; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DependResult; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; @@ -53,6 +54,7 @@ public class DependentTaskExecThread extends MasterBaseTaskExecThread { /** * dependent date */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date dependentDate; /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java index 2009c40cd7..50f8989b0e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java @@ -31,7 +31,7 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; @@ -78,7 +78,7 @@ public class TaskExecuteProcessor implements NettyRequestProcessor { Preconditions.checkArgument(CommandType.TASK_EXECUTE_REQUEST == command.getType(), String.format("invalid command type : %s", command.getType())); - TaskExecuteRequestCommand taskRequestCommand = JacksonSerializer.deserialize( + TaskExecuteRequestCommand taskRequestCommand = JsonSerializer.deserialize( command.getBody(), TaskExecuteRequestCommand.class); logger.info("received command : {}", taskRequestCommand); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java index 76323f97a8..17ae6a43b1 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java @@ -29,7 +29,7 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskKillRequestCommand; import org.apache.dolphinscheduler.remote.command.TaskKillResponseCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.Pair; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; @@ -83,7 +83,7 @@ public class TaskKillProcessor implements NettyRequestProcessor { @Override public void process(Channel channel, Command command) { Preconditions.checkArgument(CommandType.TASK_KILL_REQUEST == command.getType(), String.format("invalid command type : %s", command.getType())); - TaskKillRequestCommand killCommand = JacksonSerializer.deserialize(command.getBody(), TaskKillRequestCommand.class); + TaskKillRequestCommand killCommand = JsonSerializer.deserialize(command.getBody(), TaskKillRequestCommand.class); logger.info("received kill command : {}", killCommand); Pair> result = doKill(killCommand); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java index 17b4b17715..b964a7289b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java @@ -154,12 +154,7 @@ public class TaskExecuteThread implements Runnable { String globalParamsStr = taskExecutionContext.getGlobalParams(); if (globalParamsStr != null) { List globalParamsList = new ArrayList<>(); - - try { - globalParamsList = JSONUtils.getMapper().readValue(globalParamsStr, new TypeReference>() {}); - } catch (Exception e) { - logger.error("parse list exception!", e); - } + globalParamsList = JSONUtils.toList(globalParamsStr, Property.class); globalParamsMap.putAll(globalParamsList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue))); } return globalParamsMap; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java index 00e78d37d1..0e619eb306 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.server.worker.task; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.Direct; @@ -80,6 +81,7 @@ public class TaskProps { /** * task start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date taskStartTime; /** @@ -99,6 +101,7 @@ public class TaskProps { /** * schedule time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date scheduleTime; /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index f85dfb8a38..723ded4f2f 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -55,10 +55,7 @@ import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; import java.sql.*; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; /** @@ -278,7 +275,9 @@ public class DataxTask extends AbstractTask { String[] columns = parsingSqlColumnNames(DbType.of(dataxTaskExecutionContext.getSourcetype()), DbType.of(dataxTaskExecutionContext.getTargetType()), dataSourceCfg, dataXParameters.getSql()); + ArrayNode columnArr = writerParam.putArray("column"); + columnArr.addAll() for (String column : columns) { columnArr.add(column); } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java index 5a9ff568ae..250b2a2b0a 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java @@ -23,6 +23,7 @@ import java.util.Date; import java.util.List; import java.util.UUID; +import com.alibaba.fastjson.JSON; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.common.enums.CommandType; diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java index c1a766d531..c4fe4475cc 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java @@ -21,7 +21,7 @@ import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.log.*; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.remote.utils.JacksonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,7 +77,7 @@ public class LogClientService { Command command = request.convert2Command(); Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); if(response != null){ - RollViewLogResponseCommand rollReviewLog = JacksonSerializer.deserialize( + RollViewLogResponseCommand rollReviewLog = JsonSerializer.deserialize( response.getBody(), RollViewLogResponseCommand.class); return rollReviewLog.getMsg(); } @@ -105,7 +105,7 @@ public class LogClientService { Command command = request.convert2Command(); Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); if(response != null){ - ViewLogResponseCommand viewLog = JacksonSerializer.deserialize( + ViewLogResponseCommand viewLog = JsonSerializer.deserialize( response.getBody(), ViewLogResponseCommand.class); return viewLog.getMsg(); } @@ -133,7 +133,7 @@ public class LogClientService { Command command = request.convert2Command(); Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); if(response != null){ - GetLogBytesResponseCommand getLog = JacksonSerializer.deserialize( + GetLogBytesResponseCommand getLog = JsonSerializer.deserialize( response.getBody(), GetLogBytesResponseCommand.class); return getLog.getData(); } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index ea2fbe51fc..127369331d 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -753,15 +753,9 @@ public class ProcessService { * @return global params join */ private String joinGlobalParams(String parentGlobalParams, String subGlobalParams){ - List parentPropertyList = new ArrayList<>(); - List subPropertyList = new ArrayList<>(); - try { - parentPropertyList = JSONUtils.getMapper().readValue(parentGlobalParams, new TypeReference>() {}); - subPropertyList = JSONUtils.getMapper().readValue(subGlobalParams, new TypeReference>() {}); - } catch (IOException e) { - logger.error("json parse exception!", e); - } + List parentPropertyList = JSONUtils.toList(parentGlobalParams, Property.class); + List subPropertyList = JSONUtils.toList(subGlobalParams, Property.class); Map subMap = subPropertyList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); diff --git a/pom.xml b/pom.xml index 7c58ebc373..0742728584 100644 --- a/pom.xml +++ b/pom.xml @@ -786,7 +786,7 @@ **/dao/mapper/CommandMapperTest.java **/dao/mapper/ConnectionFactoryTest.java **/dao/mapper/DataSourceMapperTest.java - **/remote/JacksonSerializerTest.java + **/remote/JsonSerializerTest.java **/remote/NettyRemotingClientTest.java **/remote/ResponseFutureTest.java **/server/log/MasterLogFilterTest.java From 8cabb7a907f408d46b442e9f2635d6e97454aac3 Mon Sep 17 00:00:00 2001 From: simon824 Date: Tue, 9 Jun 2020 14:15:13 +0800 Subject: [PATCH 20/38] 1. specified jackson Date format. 2. remove getMapper method,replace with toList() 3. rewrite toList method,support constructCollectionType. 4. parseObject(resp, Map.class) replace to toMap() method 5. rename JacksonSerialize to JsonSerializer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rewrite createArrayNode,createObjectNode method will Affect too many lines ，i will fix it in the next pr. --- .../org/apache/dolphinscheduler/common/utils/JSONUtils.java | 4 ++-- .../dolphinscheduler/server/worker/task/datax/DataxTask.java | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index 63611cc7d6..c78ceb2d5c 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -187,7 +187,7 @@ public class JSONUtils { */ public static Map toMap(String json) { if (StringUtils.isEmpty(json)) { - return new HashMap<>(); + return null; } try { @@ -196,7 +196,7 @@ public class JSONUtils { logger.error("json to map exception!", e); } - return new HashMap<>(); + return null; } /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index 723ded4f2f..4939456ce0 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -277,7 +277,6 @@ public class DataxTask extends AbstractTask { dataSourceCfg, dataXParameters.getSql()); ArrayNode columnArr = writerParam.putArray("column"); - columnArr.addAll() for (String column : columns) { columnArr.add(column); } From c5b3f35bd4f217df31cd1fba8eb605e52909f972 Mon Sep 17 00:00:00 2001 From: simon824 Date: Tue, 9 Jun 2020 14:37:16 +0800 Subject: [PATCH 21/38] replace fastjson annotation --- .../api/dto/resources/ResourceComponent.java | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java index 3f7c227ae0..6a5b6e3353 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java @@ -1,7 +1,6 @@ package org.apache.dolphinscheduler.api.dto.resources; -import com.alibaba.fastjson.annotation.JSONType; -import com.alibaba.fastjson.annotation.JSONField; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; import org.apache.dolphinscheduler.common.enums.ResourceType; import java.util.ArrayList; @@ -26,7 +25,7 @@ import java.util.List; /** * resource component */ -@JSONType(orders={"id","pid","name","fullName","description","isDirctory","children","type"}) +@JsonPropertyOrder({"id","pid","name","fullName","description","isDirctory","children","type"}) public abstract class ResourceComponent { public ResourceComponent() { } @@ -46,17 +45,14 @@ public abstract class ResourceComponent { /** * id */ - @JSONField(ordinal = 1) protected int id; /** * parent id */ - @JSONField(ordinal = 2) protected int pid; /** * name */ - @JSONField(ordinal = 3) protected String name; /** * current directory @@ -65,32 +61,26 @@ public abstract class ResourceComponent { /** * full name */ - @JSONField(ordinal = 4) protected String fullName; /** * description */ - @JSONField(ordinal = 5) protected String description; /** * is directory */ - @JSONField(ordinal = 6) protected boolean isDirctory; /** * id value */ - @JSONField(ordinal = 7) protected String idValue; /** * resoruce type */ - @JSONField(ordinal = 8) protected ResourceType type; /** * children */ - @JSONField(ordinal = 8) protected List children = new ArrayList<>(); /** From fad09075f9dd4542ded0be6a490bfe83415dedd5 Mon Sep 17 00:00:00 2001 From: simon824 Date: Wed, 10 Jun 2020 09:16:28 +0800 Subject: [PATCH 22/38] remove fastjson maven dependence --- dolphinscheduler-alert/pom.xml | 5 ----- dolphinscheduler-api/pom.xml | 5 ----- dolphinscheduler-common/pom.xml | 6 +----- dolphinscheduler-dao/pom.xml | 6 ------ dolphinscheduler-remote/pom.xml | 4 ---- .../server/worker/task/datax/DataxTaskTest.java | 1 - pom.xml | 6 ------ 7 files changed, 1 insertion(+), 32 deletions(-) diff --git a/dolphinscheduler-alert/pom.xml b/dolphinscheduler-alert/pom.xml index 188bfa3d9e..d63e3808b4 100644 --- a/dolphinscheduler-alert/pom.xml +++ b/dolphinscheduler-alert/pom.xml @@ -66,11 +66,6 @@ commons-email - - com.alibaba - fastjson - - com.fasterxml.jackson.core jackson-core diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index b7c3f3da69..16f611fd66 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -86,11 +86,6 @@ spring-context - - com.alibaba - fastjson - - commons-collections commons-collections diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index 4d4ccede26..28acb21fd5 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -36,11 +36,7 @@ org.apache.dolphinscheduler dolphinscheduler-plugin-api - - com.alibaba - fastjson - compile - + org.apache.httpcomponents httpclient diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index 3aea888f94..b7fc8725e9 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -122,12 +122,6 @@ commons-httpclient - - com.alibaba - fastjson - compile - - com.cronutils cron-utils diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml index 66994b4076..c01434c51f 100644 --- a/dolphinscheduler-remote/pom.xml +++ b/dolphinscheduler-remote/pom.xml @@ -43,10 +43,6 @@ org.slf4j slf4j-api - - com.alibaba - fastjson - junit junit diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java index 250b2a2b0a..5a9ff568ae 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java @@ -23,7 +23,6 @@ import java.util.Date; import java.util.List; import java.util.UUID; -import com.alibaba.fastjson.JSON; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.common.enums.CommandType; diff --git a/pom.xml b/pom.xml index 861fde840b..f7c3d65671 100644 --- a/pom.xml +++ b/pom.xml @@ -69,7 +69,6 @@ 3.2.0 2.0.1 5.0.5 - 1.2.61 1.1.14 1.4.200 1.6 @@ -150,11 +149,6 @@ ${cron.utils.version} - - com.alibaba - fastjson - ${fastjson.version} - com.alibaba druid From 7d45e116bb98776c42f817c484172eabc8cac441 Mon Sep 17 00:00:00 2001 From: simon824 Date: Wed, 10 Jun 2020 09:29:39 +0800 Subject: [PATCH 23/38] remove fastjson license --- dolphinscheduler-dist/release-docs/LICENSE | 1 - dolphinscheduler-dist/release-docs/NOTICE | 20 -------------------- tools/dependencies/known-dependencies.txt | 1 - 3 files changed, 22 deletions(-) diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 52563f5573..f90a346d37 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -255,7 +255,6 @@ The text of each license is also included at licenses/LICENSE-[project].txt. derby 10.14.2.0: https://github.com/apache/derby, Apache 2.0 druid 1.1.14: https://mvnrepository.com/artifact/com.alibaba/druid/1.1.14, Apache 2.0 error_prone_annotations 2.1.2: https://mvnrepository.com/artifact/com.google.errorprone/error_prone_annotations/2.1.2, Apache 2.0 - fastjson 1.2.61: https://mvnrepository.com/artifact/com.alibaba/fastjson/1.2.61, Apache 2.0 freemarker 2.3.21: https://github.com/apache/freemarker, Apache 2.0 grpc-context 1.9.0: https://mvnrepository.com/artifact/io.grpc/grpc-context/1.9.0, Apache 2.0 grpc-core 1.9.0: https://mvnrepository.com/artifact/io.grpc/grpc-core/1.9.0, Apache 2.0 diff --git a/dolphinscheduler-dist/release-docs/NOTICE b/dolphinscheduler-dist/release-docs/NOTICE index ecab936a4f..ca2f914701 100644 --- a/dolphinscheduler-dist/release-docs/NOTICE +++ b/dolphinscheduler-dist/release-docs/NOTICE @@ -2214,26 +2214,6 @@ The Apache Software Foundation (http://www.apache.org/). ========================================================================= -FastJson NOTICE -========================================================================= -/* - * Copyright 1999-2017 Alibaba Group. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -========================================================================= - - Datanucleus NOTICE ========================================================================= diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index b1c7f5f806..f045bb275c 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -43,7 +43,6 @@ datanucleus-core-4.1.6.jar datanucleus-rdbms-4.1.7.jar derby-10.14.2.0.jar druid-1.1.14.jar -fastjson-1.2.61.jar gson-2.8.5.jar guava-20.0.jar guice-3.0.jar From f643981f489e34dff84f23ffa9d559a7e398cb4b Mon Sep 17 00:00:00 2001 From: simon824 Date: Thu, 11 Jun 2020 10:14:15 +0800 Subject: [PATCH 24/38] 1. change windows new line 'CR' to linux new line 'LF' 2. Format code --- .../remote/command/TaskExecuteAckCommand.java | 134 +++++++++++++++++- .../command/TaskExecuteRequestCommand.java | 68 ++++++++- .../command/TaskExecuteResponseCommand.java | 130 ++++++++++++++++- .../worker/runner/TaskExecuteThread.java | 3 +- .../service/process/ProcessService.java | 3 +- 5 files changed, 331 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteAckCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteAckCommand.java index 796e4dc358..135c14975b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteAckCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteAckCommand.java @@ -1 +1,133 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; import java.util.Date; /** * execute task request command */ public class TaskExecuteAckCommand implements Serializable { /** * taskInstanceId */ private int taskInstanceId; /** * startTime */ @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** * host */ private String host; /** * status */ private int status; /** * logPath */ private String logPath; /** * executePath */ private String executePath; public Date getStartTime() { return startTime; } public void setStartTime(Date startTime) { this.startTime = startTime; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } public String getLogPath() { return logPath; } public void setLogPath(String logPath) { this.logPath = logPath; } public String getExecutePath() { return executePath; } public void setExecutePath(String executePath) { this.executePath = executePath; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_EXECUTE_ACK); byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskExecuteAckCommand{" + "taskInstanceId=" + taskInstanceId + ", startTime=" + startTime + ", host='" + host + '\'' + ", status=" + status + ", logPath='" + logPath + '\'' + ", executePath='" + executePath + '\'' + '}'; } } \ No newline at end of file +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.command; + +import com.fasterxml.jackson.annotation.JsonFormat; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; + +import java.io.Serializable; +import java.util.Date; + +/** + * execute task request command + */ +public class TaskExecuteAckCommand implements Serializable { + + /** + * taskInstanceId + */ + private int taskInstanceId; + + /** + * startTime + */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + private Date startTime; + + /** + * host + */ + private String host; + + /** + * status + */ + private int status; + + /** + * logPath + */ + private String logPath; + + /** + * executePath + */ + private String executePath; + + public Date getStartTime() { + return startTime; + } + + public void setStartTime(Date startTime) { + this.startTime = startTime; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + public int getTaskInstanceId() { + return taskInstanceId; + } + + public void setTaskInstanceId(int taskInstanceId) { + this.taskInstanceId = taskInstanceId; + } + + public String getLogPath() { + return logPath; + } + + public void setLogPath(String logPath) { + this.logPath = logPath; + } + + public String getExecutePath() { + return executePath; + } + + public void setExecutePath(String executePath) { + this.executePath = executePath; + } + + /** + * package request command + * + * @return command + */ + public Command convert2Command(){ + Command command = new Command(); + command.setType(CommandType.TASK_EXECUTE_ACK); + byte[] body = JsonSerializer.serialize(this); + command.setBody(body); + return command; + } + + @Override + public String toString() { + return "TaskExecuteAckCommand{" + + "taskInstanceId=" + taskInstanceId + + ", startTime=" + startTime + + ", host='" + host + '\'' + + ", status=" + status + + ", logPath='" + logPath + '\'' + + ", executePath='" + executePath + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java index 603f8a088b..4ae28e3ca5 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java @@ -1 +1,67 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; /** * execute task request command */ public class TaskExecuteRequestCommand implements Serializable { /** * task execution context */ private String taskExecutionContext; public String getTaskExecutionContext() { return taskExecutionContext; } public void setTaskExecutionContext(String taskExecutionContext) { this.taskExecutionContext = taskExecutionContext; } public TaskExecuteRequestCommand() { } public TaskExecuteRequestCommand(String taskExecutionContext) { this.taskExecutionContext = taskExecutionContext; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_EXECUTE_REQUEST); byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskExecuteRequestCommand{" + "taskExecutionContext='" + taskExecutionContext + '\'' + '}'; } } \ No newline at end of file +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.command; + +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; + +import java.io.Serializable; + +/** + * execute task request command + */ +public class TaskExecuteRequestCommand implements Serializable { + + /** + * task execution context + */ + private String taskExecutionContext; + + public String getTaskExecutionContext() { + return taskExecutionContext; + } + + public void setTaskExecutionContext(String taskExecutionContext) { + this.taskExecutionContext = taskExecutionContext; + } + + public TaskExecuteRequestCommand() { + } + + public TaskExecuteRequestCommand(String taskExecutionContext) { + this.taskExecutionContext = taskExecutionContext; + } + + /** + * package request command + * + * @return command + */ + public Command convert2Command(){ + Command command = new Command(); + command.setType(CommandType.TASK_EXECUTE_REQUEST); + byte[] body = JsonSerializer.serialize(this); + command.setBody(body); + return command; + } + + @Override + public String toString() { + return "TaskExecuteRequestCommand{" + + "taskExecutionContext='" + taskExecutionContext + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java index 0ea34fe39f..e559334f48 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java @@ -1 +1,129 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; import java.util.Date; /** * execute task response command */ public class TaskExecuteResponseCommand implements Serializable { public TaskExecuteResponseCommand() { } public TaskExecuteResponseCommand(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } /** * task instance id */ private int taskInstanceId; /** * status */ private int status; /** * end time */ @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date endTime; /** * processId */ private int processId; /** * appIds */ private String appIds; public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public Date getEndTime() { return endTime; } public void setEndTime(Date endTime) { this.endTime = endTime; } public int getProcessId() { return processId; } public void setProcessId(int processId) { this.processId = processId; } public String getAppIds() { return appIds; } public void setAppIds(String appIds) { this.appIds = appIds; } /** * package response command * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_EXECUTE_RESPONSE); byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskExecuteResponseCommand{" + "taskInstanceId=" + taskInstanceId + ", status=" + status + ", endTime=" + endTime + ", processId=" + processId + ", appIds='" + appIds + '\'' + '}'; } } \ No newline at end of file +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.command; + +import com.fasterxml.jackson.annotation.JsonFormat; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; + +import java.io.Serializable; +import java.util.Date; + +/** + * execute task response command + */ +public class TaskExecuteResponseCommand implements Serializable { + + + public TaskExecuteResponseCommand() { + } + + public TaskExecuteResponseCommand(int taskInstanceId) { + this.taskInstanceId = taskInstanceId; + } + + /** + * task instance id + */ + private int taskInstanceId; + + /** + * status + */ + private int status; + + + /** + * end time + */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + private Date endTime; + + + /** + * processId + */ + private int processId; + + /** + * appIds + */ + private String appIds; + + + public int getTaskInstanceId() { + return taskInstanceId; + } + + public void setTaskInstanceId(int taskInstanceId) { + this.taskInstanceId = taskInstanceId; + } + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + public Date getEndTime() { + return endTime; + } + + public void setEndTime(Date endTime) { + this.endTime = endTime; + } + + public int getProcessId() { + return processId; + } + + public void setProcessId(int processId) { + this.processId = processId; + } + + public String getAppIds() { + return appIds; + } + + public void setAppIds(String appIds) { + this.appIds = appIds; + } + + /** + * package response command + * @return command + */ + public Command convert2Command(){ + Command command = new Command(); + command.setType(CommandType.TASK_EXECUTE_RESPONSE); + byte[] body = JsonSerializer.serialize(this); + command.setBody(body); + return command; + } + + @Override + public String toString() { + return "TaskExecuteResponseCommand{" + + "taskInstanceId=" + taskInstanceId + + ", status=" + status + + ", endTime=" + endTime + + ", processId=" + processId + + ", appIds='" + appIds + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java index b964a7289b..677f09e11b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java @@ -153,8 +153,7 @@ public class TaskExecuteThread implements Runnable { // global params string String globalParamsStr = taskExecutionContext.getGlobalParams(); if (globalParamsStr != null) { - List globalParamsList = new ArrayList<>(); - globalParamsList = JSONUtils.toList(globalParamsStr, Property.class); + List globalParamsList = JSONUtils.toList(globalParamsStr, Property.class); globalParamsMap.putAll(globalParamsList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue))); } return globalParamsMap; diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 127369331d..237f2a8289 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -207,14 +207,13 @@ public class ProcessService { if(cmdTypeMap.containsKey(commandType)){ ObjectNode cmdParamObj = JSONUtils.parseObject(command.getCommandParam()); - ObjectNode tempObj; int processInstanceId = cmdParamObj.path(CMDPARAM_RECOVER_PROCESS_ID_STRING).asInt(); List commands = commandMapper.selectList(null); // for all commands for (Command tmpCommand:commands){ if(cmdTypeMap.containsKey(tmpCommand.getCommandType())){ - tempObj = JSONUtils.parseObject(tmpCommand.getCommandParam()); + ObjectNode tempObj = JSONUtils.parseObject(tmpCommand.getCommandParam()); if(tempObj != null && processInstanceId == tempObj.path(CMDPARAM_RECOVER_PROCESS_ID_STRING).asInt()){ isNeedCreate = false; break; From 4e37be68af658f4b79cd9744e7e0aea8d538188f Mon Sep 17 00:00:00 2001 From: Simon <3656562@qq.com> Date: Fri, 12 Jun 2020 09:18:00 +0800 Subject: [PATCH 25/38] Update SqoopTaskTest.java --- .../worker/task/sqoop/SqoopTaskTest.java | 126 +++++++++++------- 1 file changed, 79 insertions(+), 47 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java index 2af7bd34ef..0f18b4e2ac 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java @@ -16,11 +16,10 @@ */ package org.apache.dolphinscheduler.server.worker.task.sqoop; -import org.apache.dolphinscheduler.common.enums.DbType; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; -import org.apache.dolphinscheduler.dao.entity.DataSource; +import org.apache.dolphinscheduler.server.entity.SqoopTaskExecutionContext; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; -import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.SqoopJobGenerator; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -33,7 +32,6 @@ import org.mockito.junit.MockitoJUnitRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; -import org.apache.dolphinscheduler.common.utils.*; import java.util.Date; @@ -52,64 +50,98 @@ public class SqoopTaskTest { @Before public void before() throws Exception{ processService = Mockito.mock(ProcessService.class); - Mockito.when(processService.findDataSourceById(2)).thenReturn(getDataSource()); applicationContext = Mockito.mock(ApplicationContext.class); SpringApplicationContext springApplicationContext = new SpringApplicationContext(); springApplicationContext.setApplicationContext(applicationContext); Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService); - TaskProps props = new TaskProps(); - props.setTaskAppId(String.valueOf(System.currentTimeMillis())); - props.setTenantCode("1"); - props.setEnvFile(".dolphinscheduler_env.sh"); - props.setTaskStartTime(new Date()); - props.setTaskTimeout(0); - props.setTaskParams("{\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HIVE\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"1\\\",\\\"srcQuerySql\\\":\\\"SELECT * FROM person_2\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[{\\\"prop\\\":\\\"id\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"Integer\\\"}]}\",\"targetParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal_2\\\",\\\"createHiveTable\\\":true,\\\"dropDelimiter\\\":false,\\\"hiveOverWrite\\\":true,\\\"replaceDelimiter\\\":\\\"\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-16\\\"}\",\"localParams\":[]}"); - - sqoopTask = new SqoopTask(new TaskExecutionContext(),logger); + TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); + taskExecutionContext.setTaskAppId(String.valueOf(System.currentTimeMillis())); + taskExecutionContext.setTenantCode("1"); + taskExecutionContext.setEnvFile(".dolphinscheduler_env.sh"); + taskExecutionContext.setStartTime(new Date()); + taskExecutionContext.setTaskTimeout(0); + taskExecutionContext.setTaskParams("{\"jobName\":\"sqoop_import\",\"jobType\":\"TEMPLATE\",\"concurrency\":1," + + "\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HIVE\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\"," + + "\\\"srcQueryType\\\":\\\"1\\\",\\\"srcQuerySql\\\":\\\"SELECT * FROM person_2\\\",\\\"srcColumnType\\\":\\\"0\\\"," + + "\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[]," + + "\\\"mapColumnJava\\\":[{\\\"prop\\\":\\\"id\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"Integer\\\"}]}\"" + + ",\"targetParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal_2\\\",\\\"createHiveTable\\\":true," + + "\\\"dropDelimiter\\\":false,\\\"hiveOverWrite\\\":true,\\\"replaceDelimiter\\\":\\\"\\\",\\\"hivePartitionKey\\\":\\\"date\\\"," + + "\\\"hivePartitionValue\\\":\\\"2020-02-16\\\"}\",\"localParams\":[]}"); + + sqoopTask = new SqoopTask(taskExecutionContext,logger); + //test sqoop tash init method sqoopTask.init(); } + /** + * test SqoopJobGenerator + */ @Test public void testGenerator(){ - String data1 = "{\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HDFS\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"0\\\",\\\"srcQuerySql\\\":\\\"\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[]}\",\"targetParams\":\"{\\\"targetPath\\\":\\\"/ods/tmp/test/person7\\\",\\\"deleteTargetDir\\\":true,\\\"fileType\\\":\\\"--as-textfile\\\",\\\"compressionCodec\\\":\\\"\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters1 = JSONUtils.parseObject(data1,SqoopParameters.class); + TaskExecutionContext mysqlTaskExecutionContext = getMysqlTaskExecutionContext(); + //sqoop TEMPLATE job + //import mysql to HDFS with hadoo + String mysqlToHdfs = "{\"jobName\":\"sqoop_import\",\"hadoopCustomParams\":[{\"prop\":\"mapreduce.map.memory.mb\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"4096\"}],\"sqoopAdvancedParams\":[{\"prop\":\"--direct\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"\"}]," + + "\"jobType\":\"TEMPLATE\",\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HDFS\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"0\\\",\\\"srcQuerySql\\\":\\\"\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[]}\",\"targetParams\":\"{\\\"targetPath\\\":\\\"/ods/tmp/test/person7\\\",\\\"deleteTargetDir\\\":true,\\\"fileType\\\":\\\"--as-textfile\\\",\\\"compressionCodec\\\":\\\"\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; + SqoopParameters mysqlToHdfsParams = JSON.parseObject(mysqlToHdfs,SqoopParameters.class); SqoopJobGenerator generator = new SqoopJobGenerator(); - String script = generator.generateSqoopJob(sqoopParameters1,new TaskExecutionContext()); - String expected = "sqoop import -m 1 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_2 --target-dir /ods/tmp/test/person7 --as-textfile --delete-target-dir --fields-terminated-by '@' --lines-terminated-by '\\n' --null-non-string 'NULL' --null-string 'NULL'"; - Assert.assertEquals(expected, script); - - String data2 = "{\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HDFS\",\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"exportDir\\\":\\\"/ods/tmp/test/person7\\\"}\",\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"id,name,age,sex,create_time\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":true,\\\"targetUpdateKey\\\":\\\"id\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters2 = JSONUtils.parseObject(data2,SqoopParameters.class); - - String script2 = generator.generateSqoopJob(sqoopParameters2,new TaskExecutionContext()); - String expected2 = "sqoop export -m 1 --export-dir /ods/tmp/test/person7 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --columns id,name,age,sex,create_time --fields-terminated-by '@' --lines-terminated-by '\\n' --update-key id --update-mode allowinsert"; - Assert.assertEquals(expected2, script2); - - String data3 = "{\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HIVE\",\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-17\\\"}\",\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":false,\\\"targetUpdateKey\\\":\\\"\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters3 = JSONUtils.parseObject(data3,SqoopParameters.class); - - String script3 = generator.generateSqoopJob(sqoopParameters3,new TaskExecutionContext()); - String expected3 = "sqoop export -m 1 --hcatalog-database stg --hcatalog-table person_internal --hcatalog-partition-keys date --hcatalog-partition-values 2020-02-17 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --fields-terminated-by '@' --lines-terminated-by '\\n'"; - Assert.assertEquals(expected3, script3); - - String data4 = "{\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HIVE\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"1\\\",\\\"srcQuerySql\\\":\\\"SELECT * FROM person_2\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[{\\\"prop\\\":\\\"id\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"Integer\\\"}]}\",\"targetParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal_2\\\",\\\"createHiveTable\\\":true,\\\"dropDelimiter\\\":false,\\\"hiveOverWrite\\\":true,\\\"replaceDelimiter\\\":\\\"\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-16\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters4 = JSONUtils.parseObject(data4,SqoopParameters.class); - - String script4 = generator.generateSqoopJob(sqoopParameters4,new TaskExecutionContext()); - String expected4 = "sqoop import -m 1 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --query 'SELECT * FROM person_2 WHERE $CONDITIONS' --map-column-java id=Integer --hive-import --hive-table stg.person_internal_2 --create-hive-table --hive-overwrite -delete-target-dir --hive-partition-key date --hive-partition-value 2020-02-16"; - Assert.assertEquals(expected4, script4); + String mysqlToHdfsScript = generator.generateSqoopJob(mysqlToHdfsParams,mysqlTaskExecutionContext); + String mysqlToHdfsExpected = "sqoop import -D mapred.job.name=sqoop_import -D mapreduce.map.memory.mb=4096 --direct -m 1 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_2 --target-dir /ods/tmp/test/person7 --as-textfile --delete-target-dir --fields-terminated-by '@' --lines-terminated-by '\\n' --null-non-string 'NULL' --null-string 'NULL'"; + Assert.assertEquals(mysqlToHdfsExpected, mysqlToHdfsScript); + + //export hdfs to mysql using update mode + String hdfsToMysql = "{\"jobName\":\"sqoop_import\",\"jobType\":\"TEMPLATE\",\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HDFS\"," + + "\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"exportDir\\\":\\\"/ods/tmp/test/person7\\\"}\"," + + "\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"id,name,age,sex,create_time\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":true,\\\"targetUpdateKey\\\":\\\"id\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; + SqoopParameters hdfsToMysqlParams = JSON.parseObject(hdfsToMysql,SqoopParameters.class); + String hdfsToMysqlScript = generator.generateSqoopJob(hdfsToMysqlParams,mysqlTaskExecutionContext); + String hdfsToMysqlScriptExpected = "sqoop export -D mapred.job.name=sqoop_import -m 1 --export-dir /ods/tmp/test/person7 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --columns id,name,age,sex,create_time --fields-terminated-by '@' --lines-terminated-by '\\n' --update-key id --update-mode allowinsert"; + Assert.assertEquals(hdfsToMysqlScriptExpected, hdfsToMysqlScript); + + //export hive to mysql + String hiveToMysql = "{\"jobName\":\"sqoop_import\",\"jobType\":\"TEMPLATE\",\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HIVE\",\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-17\\\"}\",\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":false,\\\"targetUpdateKey\\\":\\\"\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; + SqoopParameters hiveToMysqlParams = JSON.parseObject(hiveToMysql,SqoopParameters.class); + String hiveToMysqlScript = generator.generateSqoopJob(hiveToMysqlParams,mysqlTaskExecutionContext); + String hiveToMysqlExpected = "sqoop export -D mapred.job.name=sqoop_import -m 1 --hcatalog-database stg --hcatalog-table person_internal --hcatalog-partition-keys date --hcatalog-partition-values 2020-02-17 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --fields-terminated-by '@' --lines-terminated-by '\\n'"; + Assert.assertEquals(hiveToMysqlExpected, hiveToMysqlScript); + + //import mysql to hive + String mysqlToHive = "{\"jobName\":\"sqoop_import\",\"jobType\":\"TEMPLATE\",\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HIVE\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"1\\\",\\\"srcQuerySql\\\":\\\"SELECT * FROM person_2\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[{\\\"prop\\\":\\\"id\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"Integer\\\"}]}\",\"targetParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal_2\\\",\\\"createHiveTable\\\":true,\\\"dropDelimiter\\\":false,\\\"hiveOverWrite\\\":true,\\\"replaceDelimiter\\\":\\\"\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-16\\\"}\",\"localParams\":[]}"; + SqoopParameters mysqlToHiveParams = JSON.parseObject(mysqlToHive,SqoopParameters.class); + String mysqlToHiveScript = generator.generateSqoopJob(mysqlToHiveParams,mysqlTaskExecutionContext); + String mysqlToHiveExpected = "sqoop import -D mapred.job.name=sqoop_import -m 1 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --query 'SELECT * FROM person_2 WHERE $CONDITIONS' --map-column-java id=Integer --hive-import --hive-table stg.person_internal_2 --create-hive-table --hive-overwrite -delete-target-dir --hive-partition-key date --hive-partition-value 2020-02-16"; + Assert.assertEquals(mysqlToHiveExpected, mysqlToHiveScript); + + //sqoop CUSTOM job + String sqoopCustomString = "{\"jobType\":\"CUSTOM\",\"localParams\":[],\"customShell\":\"sqoop import\"}"; + SqoopParameters sqoopCustomParams = JSON.parseObject(sqoopCustomString, SqoopParameters.class); + String sqoopCustomScript = generator.generateSqoopJob(sqoopCustomParams, new TaskExecutionContext()); + String sqoopCustomExpected = "sqoop import"; + Assert.assertEquals(sqoopCustomExpected, sqoopCustomScript); } - private DataSource getDataSource() { - DataSource dataSource = new DataSource(); - dataSource.setType(DbType.MYSQL); - dataSource.setConnectionParams( - "{\"address\":\"jdbc:mysql://192.168.0.111:3306\",\"database\":\"test\",\"jdbcUrl\":\"jdbc:mysql://192.168.0.111:3306/test\",\"user\":\"kylo\",\"password\":\"123456\"}"); - dataSource.setUserId(1); - return dataSource; + + /** + * get taskExecutionContext include mysql + * @return TaskExecutionContext + */ + private TaskExecutionContext getMysqlTaskExecutionContext() { + TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); + SqoopTaskExecutionContext sqoopTaskExecutionContext = new SqoopTaskExecutionContext(); + String mysqlSourceConnectionParams = "{\"address\":\"jdbc:mysql://192.168.0.111:3306\",\"database\":\"test\",\"jdbcUrl\":\"jdbc:mysql://192.168.0.111:3306/test\",\"user\":\"kylo\",\"password\":\"123456\"}"; + String mysqlTargetConnectionParams = "{\"address\":\"jdbc:mysql://192.168.0.111:3306\",\"database\":\"test\",\"jdbcUrl\":\"jdbc:mysql://192.168.0.111:3306/test\",\"user\":\"kylo\",\"password\":\"123456\"}"; + sqoopTaskExecutionContext.setDataSourceId(2); + sqoopTaskExecutionContext.setDataTargetId(2); + sqoopTaskExecutionContext.setSourcetype(0); + sqoopTaskExecutionContext.setTargetConnectionParams(mysqlTargetConnectionParams); + sqoopTaskExecutionContext.setSourceConnectionParams(mysqlSourceConnectionParams); + sqoopTaskExecutionContext.setTargetType(0); + taskExecutionContext.setSqoopTaskExecutionContext(sqoopTaskExecutionContext); + return taskExecutionContext; } @Test From 094d09ba74062dc8a828d87ba8e9d3d9a14c42a8 Mon Sep 17 00:00:00 2001 From: Simon <3656562@qq.com> Date: Fri, 12 Jun 2020 09:20:28 +0800 Subject: [PATCH 26/38] Update pom.xml --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index f7c3d65671..30c4f441ab 100644 --- a/pom.xml +++ b/pom.xml @@ -816,6 +816,7 @@ **/server/worker/task/EnvFileTest.java **/server/worker/task/spark/SparkTaskTest.java **/server/worker/task/datax/DataxTaskTest.java + **/server/worker/task/datax/SqoopTaskTest.java **/server/worker/EnvFileTest.java **/service/quartz/cron/CronUtilsTest.java **/service/zk/DefaultEnsembleProviderTest.java From b5a296f4cdf991ec22544ef1cde8b8dbf8a2cdd5 Mon Sep 17 00:00:00 2001 From: Simon <3656562@qq.com> Date: Fri, 12 Jun 2020 09:21:07 +0800 Subject: [PATCH 27/38] Update pom.xml --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index 30c4f441ab..d71e98e32c 100644 --- a/pom.xml +++ b/pom.xml @@ -815,7 +815,6 @@ **/server/worker/task/spark/SparkTaskTest.java **/server/worker/task/EnvFileTest.java **/server/worker/task/spark/SparkTaskTest.java - **/server/worker/task/datax/DataxTaskTest.java **/server/worker/task/datax/SqoopTaskTest.java **/server/worker/EnvFileTest.java **/service/quartz/cron/CronUtilsTest.java From 71619ed29bc9ff3f34a5308576711d86779392a6 Mon Sep 17 00:00:00 2001 From: Simon <3656562@qq.com> Date: Fri, 12 Jun 2020 09:22:44 +0800 Subject: [PATCH 28/38] Update pom.xml --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d71e98e32c..0e625ea9a7 100644 --- a/pom.xml +++ b/pom.xml @@ -815,7 +815,7 @@ **/server/worker/task/spark/SparkTaskTest.java **/server/worker/task/EnvFileTest.java **/server/worker/task/spark/SparkTaskTest.java - **/server/worker/task/datax/SqoopTaskTest.java + **/server/worker/task/sqoop/SqoopTaskTest.java **/server/worker/EnvFileTest.java **/service/quartz/cron/CronUtilsTest.java **/service/zk/DefaultEnsembleProviderTest.java From 65841a5709aa681e7b3df6d8cdc10b575fdb9d7b Mon Sep 17 00:00:00 2001 From: simon824 Date: Fri, 12 Jun 2020 09:31:04 +0800 Subject: [PATCH 29/38] 1. change windows new line 'CR' to linux new line 'LF' 2. solve conflict --- .../command/TaskKillRequestCommand.java | 62 ++++++++- .../command/TaskKillResponseCommand.java | 120 +++++++++++++++++- .../worker/task/sqoop/SqoopTaskTest.java | 12 +- pom.xml | 1 + 4 files changed, 187 insertions(+), 8 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java index 092e462a1f..4c0830b7cf 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java @@ -1 +1,61 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; /** * kill task request command */ public class TaskKillRequestCommand implements Serializable { /** * task id */ private int taskInstanceId; public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_KILL_REQUEST); byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskKillRequestCommand{" + "taskInstanceId=" + taskInstanceId + '}'; } } \ No newline at end of file +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.command; + +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; + +import java.io.Serializable; + +/** + * kill task request command + */ +public class TaskKillRequestCommand implements Serializable { + + /** + * task id + */ + private int taskInstanceId; + + + public int getTaskInstanceId() { + return taskInstanceId; + } + + public void setTaskInstanceId(int taskInstanceId) { + this.taskInstanceId = taskInstanceId; + } + + /** + * package request command + * + * @return command + */ + public Command convert2Command(){ + Command command = new Command(); + command.setType(CommandType.TASK_KILL_REQUEST); + byte[] body = JsonSerializer.serialize(this); + command.setBody(body); + return command; + } + + @Override + public String toString() { + return "TaskKillRequestCommand{" + + "taskInstanceId=" + taskInstanceId + + '}'; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java index 9dc0c541d7..4b48c1ef4f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java @@ -1 +1,119 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; import java.util.List; /** * kill task response command */ public class TaskKillResponseCommand implements Serializable { /** * taskInstanceId */ private int taskInstanceId; /** * host */ private String host; /** * status */ private int status; /** * processId */ private int processId; /** * other resource manager appId , for example : YARN etc */ protected List appIds; public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public int getProcessId() { return processId; } public void setProcessId(int processId) { this.processId = processId; } public List getAppIds() { return appIds; } public void setAppIds(List appIds) { this.appIds = appIds; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_KILL_RESPONSE); byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskKillResponseCommand{" + "taskInstanceId=" + taskInstanceId + ", host='" + host + '\'' + ", status=" + status + ", processId=" + processId + ", appIds=" + appIds + '}'; } } \ No newline at end of file +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.command; + +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; + +import java.io.Serializable; +import java.util.List; + +/** + * kill task response command + */ +public class TaskKillResponseCommand implements Serializable { + + /** + * taskInstanceId + */ + private int taskInstanceId; + + /** + * host + */ + private String host; + + /** + * status + */ + private int status; + + + /** + * processId + */ + private int processId; + + /** + * other resource manager appId , for example : YARN etc + */ + protected List appIds; + + + public int getTaskInstanceId() { + return taskInstanceId; + } + + public void setTaskInstanceId(int taskInstanceId) { + this.taskInstanceId = taskInstanceId; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + public int getProcessId() { + return processId; + } + + public void setProcessId(int processId) { + this.processId = processId; + } + + public List getAppIds() { + return appIds; + } + + public void setAppIds(List appIds) { + this.appIds = appIds; + } + + /** + * package request command + * + * @return command + */ + public Command convert2Command(){ + Command command = new Command(); + command.setType(CommandType.TASK_KILL_RESPONSE); + byte[] body = JsonSerializer.serialize(this); + command.setBody(body); + return command; + } + + @Override + public String toString() { + return "TaskKillResponseCommand{" + + "taskInstanceId=" + taskInstanceId + + ", host='" + host + '\'' + + ", status=" + status + + ", processId=" + processId + + ", appIds=" + appIds + + '}'; + } +} diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java index 0f18b4e2ac..2d0e39aa69 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java @@ -16,8 +16,8 @@ */ package org.apache.dolphinscheduler.server.worker.task.sqoop; -import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.server.entity.SqoopTaskExecutionContext; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.SqoopJobGenerator; @@ -86,7 +86,7 @@ public class SqoopTaskTest { //import mysql to HDFS with hadoo String mysqlToHdfs = "{\"jobName\":\"sqoop_import\",\"hadoopCustomParams\":[{\"prop\":\"mapreduce.map.memory.mb\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"4096\"}],\"sqoopAdvancedParams\":[{\"prop\":\"--direct\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"\"}]," + "\"jobType\":\"TEMPLATE\",\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HDFS\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"0\\\",\\\"srcQuerySql\\\":\\\"\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[]}\",\"targetParams\":\"{\\\"targetPath\\\":\\\"/ods/tmp/test/person7\\\",\\\"deleteTargetDir\\\":true,\\\"fileType\\\":\\\"--as-textfile\\\",\\\"compressionCodec\\\":\\\"\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters mysqlToHdfsParams = JSON.parseObject(mysqlToHdfs,SqoopParameters.class); + SqoopParameters mysqlToHdfsParams = JSONUtils.parseObject(mysqlToHdfs,SqoopParameters.class); SqoopJobGenerator generator = new SqoopJobGenerator(); String mysqlToHdfsScript = generator.generateSqoopJob(mysqlToHdfsParams,mysqlTaskExecutionContext); String mysqlToHdfsExpected = "sqoop import -D mapred.job.name=sqoop_import -D mapreduce.map.memory.mb=4096 --direct -m 1 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_2 --target-dir /ods/tmp/test/person7 --as-textfile --delete-target-dir --fields-terminated-by '@' --lines-terminated-by '\\n' --null-non-string 'NULL' --null-string 'NULL'"; @@ -96,28 +96,28 @@ public class SqoopTaskTest { String hdfsToMysql = "{\"jobName\":\"sqoop_import\",\"jobType\":\"TEMPLATE\",\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HDFS\"," + "\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"exportDir\\\":\\\"/ods/tmp/test/person7\\\"}\"," + "\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"id,name,age,sex,create_time\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":true,\\\"targetUpdateKey\\\":\\\"id\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters hdfsToMysqlParams = JSON.parseObject(hdfsToMysql,SqoopParameters.class); + SqoopParameters hdfsToMysqlParams = JSONUtils.parseObject(hdfsToMysql,SqoopParameters.class); String hdfsToMysqlScript = generator.generateSqoopJob(hdfsToMysqlParams,mysqlTaskExecutionContext); String hdfsToMysqlScriptExpected = "sqoop export -D mapred.job.name=sqoop_import -m 1 --export-dir /ods/tmp/test/person7 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --columns id,name,age,sex,create_time --fields-terminated-by '@' --lines-terminated-by '\\n' --update-key id --update-mode allowinsert"; Assert.assertEquals(hdfsToMysqlScriptExpected, hdfsToMysqlScript); //export hive to mysql String hiveToMysql = "{\"jobName\":\"sqoop_import\",\"jobType\":\"TEMPLATE\",\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HIVE\",\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-17\\\"}\",\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":false,\\\"targetUpdateKey\\\":\\\"\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters hiveToMysqlParams = JSON.parseObject(hiveToMysql,SqoopParameters.class); + SqoopParameters hiveToMysqlParams = JSONUtils.parseObject(hiveToMysql,SqoopParameters.class); String hiveToMysqlScript = generator.generateSqoopJob(hiveToMysqlParams,mysqlTaskExecutionContext); String hiveToMysqlExpected = "sqoop export -D mapred.job.name=sqoop_import -m 1 --hcatalog-database stg --hcatalog-table person_internal --hcatalog-partition-keys date --hcatalog-partition-values 2020-02-17 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --fields-terminated-by '@' --lines-terminated-by '\\n'"; Assert.assertEquals(hiveToMysqlExpected, hiveToMysqlScript); //import mysql to hive String mysqlToHive = "{\"jobName\":\"sqoop_import\",\"jobType\":\"TEMPLATE\",\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HIVE\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"1\\\",\\\"srcQuerySql\\\":\\\"SELECT * FROM person_2\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[{\\\"prop\\\":\\\"id\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"Integer\\\"}]}\",\"targetParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal_2\\\",\\\"createHiveTable\\\":true,\\\"dropDelimiter\\\":false,\\\"hiveOverWrite\\\":true,\\\"replaceDelimiter\\\":\\\"\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-16\\\"}\",\"localParams\":[]}"; - SqoopParameters mysqlToHiveParams = JSON.parseObject(mysqlToHive,SqoopParameters.class); + SqoopParameters mysqlToHiveParams = JSONUtils.parseObject(mysqlToHive,SqoopParameters.class); String mysqlToHiveScript = generator.generateSqoopJob(mysqlToHiveParams,mysqlTaskExecutionContext); String mysqlToHiveExpected = "sqoop import -D mapred.job.name=sqoop_import -m 1 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --query 'SELECT * FROM person_2 WHERE $CONDITIONS' --map-column-java id=Integer --hive-import --hive-table stg.person_internal_2 --create-hive-table --hive-overwrite -delete-target-dir --hive-partition-key date --hive-partition-value 2020-02-16"; Assert.assertEquals(mysqlToHiveExpected, mysqlToHiveScript); //sqoop CUSTOM job String sqoopCustomString = "{\"jobType\":\"CUSTOM\",\"localParams\":[],\"customShell\":\"sqoop import\"}"; - SqoopParameters sqoopCustomParams = JSON.parseObject(sqoopCustomString, SqoopParameters.class); + SqoopParameters sqoopCustomParams = JSONUtils.parseObject(sqoopCustomString, SqoopParameters.class); String sqoopCustomScript = generator.generateSqoopJob(sqoopCustomParams, new TaskExecutionContext()); String sqoopCustomExpected = "sqoop import"; Assert.assertEquals(sqoopCustomExpected, sqoopCustomScript); diff --git a/pom.xml b/pom.xml index 0e625ea9a7..dacc4280c0 100644 --- a/pom.xml +++ b/pom.xml @@ -815,6 +815,7 @@ **/server/worker/task/spark/SparkTaskTest.java **/server/worker/task/EnvFileTest.java **/server/worker/task/spark/SparkTaskTest.java + **/server/worker/task/datax/DataxTaskTest.java **/server/worker/task/sqoop/SqoopTaskTest.java **/server/worker/EnvFileTest.java **/service/quartz/cron/CronUtilsTest.java From cfecfb799b91459e825c2104ab6fa25d07e4f7f1 Mon Sep 17 00:00:00 2001 From: simon824 Date: Fri, 12 Jun 2020 09:59:40 +0800 Subject: [PATCH 30/38] 1. change windows new line 'CR' to linux new line 'LF' 2. solve conflict --- .../org/apache/dolphinscheduler/common/utils/JSONUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index c78ceb2d5c..20209c0a98 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -119,7 +119,7 @@ public class JSONUtils { */ public static List toList(String json, Class clazz) { if (StringUtils.isEmpty(json)) { - return new ArrayList<>(); + return Collections.emptyList(); } try { @@ -130,7 +130,7 @@ public class JSONUtils { logger.error("parse list exception!", e); } - return new ArrayList<>(); + return Collections.emptyList(); } From 24120a4641c014a4c9dcdb36cfa787f2b660822e Mon Sep 17 00:00:00 2001 From: simon824 Date: Fri, 12 Jun 2020 11:27:42 +0800 Subject: [PATCH 31/38] fix the bad code smell --- .../alert/utils/EnterpriseWeChatUtils.java | 1 - .../api/service/DataSourceService.java | 2 +- .../api/service/ProcessDefinitionService.java | 1 - .../api/service/ProcessInstanceService.java | 4 +--- .../dolphinscheduler/api/service/ResourcesService.java | 4 ++-- .../dolphinscheduler/common/utils/HadoopUtils.java | 7 ++++--- .../apache/dolphinscheduler/dao/entity/AlertGroup.java | 1 - .../dolphinscheduler/dao/entity/ProcessDefinition.java | 10 +--------- .../dao/entity/ProcessDefinitionTest.java | 2 +- .../dolphinscheduler/remote/utils/JsonSerializer.java | 3 +++ .../server/worker/runner/TaskExecuteThread.java | 2 -- .../server/worker/task/datax/DataxTask.java | 10 +++++----- .../server/worker/task/sql/SqlTask.java | 7 ++++--- .../service/process/ProcessService.java | 2 -- 14 files changed, 22 insertions(+), 34 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index bebb70bae7..38dc38b4a8 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -20,7 +20,6 @@ import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.*; -import com.google.common.reflect.TypeToken; import org.apache.dolphinscheduler.plugin.model.AlertData; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index 735cd8ce89..1bfacfb93d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -539,7 +539,7 @@ public class DataSourceService extends BaseService{ } if(logger.isDebugEnabled()){ - logger.info("parameters map-----" + JSONUtils.toJsonString(parameterMap)); + logger.info("parameters map:{}", JSONUtils.toJsonString(parameterMap)); } return JSONUtils.toJsonString(parameterMap); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index c3d8516ed0..e23ed0002a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.service; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java index 5d5eb006f8..8b393a7fe6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.api.service; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import com.fasterxml.jackson.core.type.TypeReference; import org.apache.dolphinscheduler.api.dto.gantt.GanttDto; import org.apache.dolphinscheduler.api.dto.gantt.Task; import org.apache.dolphinscheduler.api.enums.Status; @@ -554,8 +553,7 @@ public class ProcessInstanceService extends BaseDAGService { String localParams = map.get(LOCAL_PARAMS); if (localParams != null && !localParams.isEmpty()) { localParams = ParameterUtils.convertParameterPlaceholders(localParams, timeParams); - List localParamsList = new ArrayList<>(); - localParamsList = JSONUtils.toList(localParams, Property.class); + List localParamsList = JSONUtils.toList(localParams, Property.class); Map localParamsMap = new HashMap<>(); localParamsMap.put("taskType",taskNode.getType()); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index bdcc38401c..7043bc0116 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -543,7 +543,6 @@ public class ResourcesService extends BaseService { } List allResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0); Visitor resourceTreeVisitor = new ResourceTreeVisitor(allResourceList); - //JSONArray jsonArray = JSONUtils.toList(JSONUtils.toJsonString(resourceTreeVisitor.visit().getChildren(), SerializerFeature.SortField)); result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren()); putMsg(result,Status.SUCCESS); @@ -1127,7 +1126,8 @@ public class ResourcesService extends BaseService { } List authedResources = resourcesMapper.queryAuthorizedResourceList(userId); Visitor visitor = new ResourceTreeVisitor(authedResources); - logger.info(JSONUtils.toJsonString(visitor.visit(), SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS)); + String visit = JSONUtils.toJsonString(visitor.visit(), SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); + logger.info(visit); String jsonTreeStr = JSONUtils.toJsonString(visitor.visit().getChildren(), SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); logger.info(jsonTreeStr); result.put(Constants.DATA_LIST, visitor.visit().getChildren()); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index e57201d74d..94f83acb57 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -16,7 +16,8 @@ */ package org.apache.dolphinscheduler.common.utils; -import com.fasterxml.jackson.core.io.JsonEOFException; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -408,7 +409,7 @@ public class HadoopUtils implements Closeable { * @param applicationId application id * @return the return may be null or there may be other parse exceptions */ - public ExecutionStatus getApplicationStatus(String applicationId) throws NullPointerException { + public ExecutionStatus getApplicationStatus(String applicationId) { if (StringUtils.isEmpty(applicationId)) { return null; } @@ -428,7 +429,7 @@ public class HadoopUtils implements Closeable { responseContent = HttpUtils.get(jobHistoryUrl); ObjectNode jsonObject = JSONUtils.parseObject(responseContent); if (!jsonObject.has("job")){ - throw new NullPointerException(); + return ExecutionStatus.FAILURE; } result = jsonObject.path("job").path("state").asText(); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AlertGroup.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AlertGroup.java index 652d63bc70..e35e2f759c 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AlertGroup.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AlertGroup.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.dao.entity; import com.baomidou.mybatisplus.annotation.TableField; -import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.AlertType; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java index 6b4a83dfd8..3648482996 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java @@ -22,21 +22,16 @@ import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.core.toolkit.StringUtils; import com.fasterxml.jackson.annotation.JsonFormat; -import com.fasterxml.jackson.core.type.TypeReference; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.utils.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import java.util.stream.Stream; /** @@ -44,7 +39,6 @@ import java.util.stream.Stream; */ @TableName("t_ds_process_definition") public class ProcessDefinition { - private static final Logger logger = LoggerFactory.getLogger(ProcessDefinition.class); /** * id @@ -300,10 +294,8 @@ public class ProcessDefinition { } public Map getGlobalParamMap() { - List propList = new ArrayList<> (); - if (globalParamMap == null && StringUtils.isNotEmpty(globalParams)) { - propList = JSONUtils.toList(globalParams,Property.class); + List propList = JSONUtils.toList(globalParams,Property.class); globalParamMap = propList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java index b342f01743..951311672a 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java @@ -32,7 +32,7 @@ public class ProcessDefinitionTest { taskInstance.setGlobalParams("[{\"prop\":\"selenium_global_parameters_1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"selenium_global_parameters_value_1\"}]"); taskInstance.getGlobalParamMap(); - Assert.assertEquals(taskInstance.getGlobalParamMap().toString(),"{selenium_global_parameters_1=selenium_global_parameters_value_1}"); + Assert.assertEquals("{selenium_global_parameters_1=selenium_global_parameters_value_1}",taskInstance.getGlobalParamMap().toString()); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java index a23dec4a75..0c05232dd1 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java @@ -31,6 +31,9 @@ public class JsonSerializer { private static final ObjectMapper objectMapper = new ObjectMapper(); private static final Logger logger = LoggerFactory.getLogger(JsonSerializer.class); + private JsonSerializer(){ + + } /** * serialize to byte * diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java index 677f09e11b..b6ab894020 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.server.worker.runner; -import com.fasterxml.jackson.core.type.TypeReference; import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.Constants; @@ -25,7 +24,6 @@ import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index 4939456ce0..5a8b8d6706 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -223,7 +223,7 @@ public class DataxTask extends AbstractTask { * @return collection of datax job config JSONObject * @throws SQLException if error throws SQLException */ - private List buildDataxJobContentJson() throws SQLException { + private List buildDataxJobContentJson() { DataxTaskExecutionContext dataxTaskExecutionContext = taskExecutionContext.getDataxTaskExecutionContext(); @@ -304,8 +304,8 @@ public class DataxTask extends AbstractTask { List contentList = new ArrayList<>(); ObjectNode content = JSONUtils.createObjectNode(); - content.put("reader", reader); - content.put("writer", writer); + content.put("reader", reader.toString()); + content.put("writer", reader.toString()); contentList.add(content); return contentList; @@ -335,8 +335,8 @@ public class DataxTask extends AbstractTask { errorLimit.put("percentage", 0); ObjectNode setting = JSONUtils.createObjectNode(); - setting.put("speed", speed); - setting.put("errorLimit", errorLimit); + setting.put("speed", speed.toString()); + setting.put("errorLimit", errorLimit.toString()); return setting; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index 3384ed17c7..d3df2287de 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -269,16 +269,17 @@ public class SqlTask extends AbstractTask { resultJSONArray.add(mapOfColValues); rowCount++; } - logger.debug("execute sql : {}", JSONUtils.toJsonString(resultJSONArray, SerializationFeature.WRITE_NULL_MAP_VALUES)); + String result = JSONUtils.toJsonString(resultJSONArray); + logger.debug("execute sql : {}", result); // if there is a result set if (!resultJSONArray.isEmpty(null) ) { if (StringUtils.isNotEmpty(sqlParameters.getTitle())) { sendAttachment(sqlParameters.getTitle(), - JSONUtils.toJsonString(resultJSONArray, SerializationFeature.WRITE_NULL_MAP_VALUES)); + JSONUtils.toJsonString(resultJSONArray)); }else{ sendAttachment(taskExecutionContext.getTaskName() + " query resultsets ", - JSONUtils.toJsonString(resultJSONArray, SerializationFeature.WRITE_NULL_MAP_VALUES)); + JSONUtils.toJsonString(resultJSONArray)); } } } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 237f2a8289..60c17f7f18 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.service.process; import com.cronutils.model.Cron; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.lang.ArrayUtils; import org.apache.dolphinscheduler.common.Constants; @@ -37,7 +36,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; -import java.io.IOException; import java.util.*; import java.util.stream.Collectors; From 65953fe2573d4358c4bbc023cf8f04f6343345bb Mon Sep 17 00:00:00 2001 From: simon824 Date: Fri, 12 Jun 2020 12:10:17 +0800 Subject: [PATCH 32/38] fix testBuildDataxJobContentJson --- .../server/worker/task/datax/DataxTask.java | 2 +- .../server/worker/task/datax/DataxTaskTest.java | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index 5a8b8d6706..a14a5d64f5 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -305,7 +305,7 @@ public class DataxTask extends AbstractTask { List contentList = new ArrayList<>(); ObjectNode content = JSONUtils.createObjectNode(); content.put("reader", reader.toString()); - content.put("writer", reader.toString()); + content.put("writer", writer.toString()); contentList.add(content); return contentList; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java index 5a9ff568ae..6c3cad5671 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java @@ -25,9 +25,7 @@ import java.util.UUID; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.task.datax.DataxParameters; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; @@ -316,13 +314,13 @@ public class DataxTaskTest { Assert.assertNotNull(contentList); ObjectNode content = contentList.get(0); - JsonNode reader = content.path("reader"); + JsonNode reader = JSONUtils.parseObject(content.path("reader").asText()); Assert.assertNotNull(reader); String readerPluginName = reader.path("name").asText(); Assert.assertEquals(DataxUtils.DATAX_READER_PLUGIN_MYSQL, readerPluginName); - JsonNode writer = content.path("writer"); + JsonNode writer = JSONUtils.parseObject(content.path("writer").asText()); Assert.assertNotNull(writer); String writerPluginName = writer.path("name").asText(); From 1b7d8ee236990747ecfc40726092cb352cd91f36 Mon Sep 17 00:00:00 2001 From: simon824 Date: Fri, 12 Jun 2020 12:35:18 +0800 Subject: [PATCH 33/38] fix the bad code smell --- .../org/apache/dolphinscheduler/common/utils/HadoopUtils.java | 2 -- .../apache/dolphinscheduler/server/worker/task/sql/SqlTask.java | 2 -- 2 files changed, 4 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 94f83acb57..71bbf1ad94 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -16,8 +16,6 @@ */ package org.apache.dolphinscheduler.common.utils; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index d3df2287de..d201a1f261 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.server.worker.task.sql; -import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.lang.StringUtils; @@ -43,7 +42,6 @@ import org.apache.dolphinscheduler.server.utils.UDFUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.slf4j.Logger; -import org.apache.dolphinscheduler.common.utils.*; import java.sql.*; import java.util.*; From 7f725c32b54344d73ea5668a8871daa5dbda4b69 Mon Sep 17 00:00:00 2001 From: Simon <3656562@qq.com> Date: Sat, 13 Jun 2020 09:24:16 +0800 Subject: [PATCH 34/38] Update pom.xml --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index dacc4280c0..1a33755377 100644 --- a/pom.xml +++ b/pom.xml @@ -816,6 +816,7 @@ **/server/worker/task/EnvFileTest.java **/server/worker/task/spark/SparkTaskTest.java **/server/worker/task/datax/DataxTaskTest.java + **/server/worker/task/http/HttpTaskTest.java **/server/worker/task/sqoop/SqoopTaskTest.java **/server/worker/EnvFileTest.java **/service/quartz/cron/CronUtilsTest.java From 6c5b0706f52478e756f91d92c9ee63314e594ea2 Mon Sep 17 00:00:00 2001 From: Simon <3656562@qq.com> Date: Sat, 13 Jun 2020 09:25:56 +0800 Subject: [PATCH 35/38] Update pom.xml --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1a33755377..d56f40b601 100644 --- a/pom.xml +++ b/pom.xml @@ -815,7 +815,6 @@ **/server/worker/task/spark/SparkTaskTest.java **/server/worker/task/EnvFileTest.java **/server/worker/task/spark/SparkTaskTest.java - **/server/worker/task/datax/DataxTaskTest.java **/server/worker/task/http/HttpTaskTest.java **/server/worker/task/sqoop/SqoopTaskTest.java **/server/worker/EnvFileTest.java From 2e4adbe03f930406d59be1c844f2ca995ee9f439 Mon Sep 17 00:00:00 2001 From: Simon <3656562@qq.com> Date: Sat, 13 Jun 2020 09:27:27 +0800 Subject: [PATCH 36/38] Update pom.xml --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 52b088f0ae..bdac1f16aa 100644 --- a/pom.xml +++ b/pom.xml @@ -816,6 +816,7 @@ **/server/worker/task/spark/SparkTaskTest.java **/server/worker/task/EnvFileTest.java **/server/worker/task/spark/SparkTaskTest.java + **/server/worker/task/datax/DataxTaskTest.java **/server/worker/task/http/HttpTaskTest.java **/server/worker/task/sqoop/SqoopTaskTest.java **/server/worker/EnvFileTest.java From 3fa4dd702b6601f6d829b2579177dae229d1ad0f Mon Sep 17 00:00:00 2001 From: simon824 Date: Sat, 13 Jun 2020 09:47:18 +0800 Subject: [PATCH 37/38] Resolved the conflict --- .../common/task/HttpParametersTest.java | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/HttpParametersTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/HttpParametersTest.java index abcfca83dd..6bd3bc76b5 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/HttpParametersTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/HttpParametersTest.java @@ -16,20 +16,13 @@ */ package org.apache.dolphinscheduler.common.task; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.enums.HttpCheckCondition; import org.apache.dolphinscheduler.common.enums.HttpMethod; -import org.apache.dolphinscheduler.common.process.HttpProperty; -import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.http.HttpParameters; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.junit.Assert; import org.junit.Test; -import com.alibaba.fastjson.JSON; - /** * http parameter */ @@ -40,7 +33,7 @@ public class HttpParametersTest { public void testGenerator(){ String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," + "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}"; - HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class); + HttpParameters httpParameters = JSONUtils.parseObject(paramData, HttpParameters.class); Assert.assertEquals(10000,httpParameters.getConnectTimeout() ); @@ -57,7 +50,7 @@ public class HttpParametersTest { public void testCheckParameters(){ String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," + "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}"; - HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class); + HttpParameters httpParameters = JSONUtils.parseObject(paramData, HttpParameters.class); Assert.assertTrue( httpParameters.checkParameters()); Assert.assertEquals(10000,httpParameters.getConnectTimeout() ); @@ -74,7 +67,7 @@ public class HttpParametersTest { public void testCheckValues() { String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," + "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}"; - HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class); + HttpParameters httpParameters = JSONUtils.parseObject(paramData, HttpParameters.class); Assert.assertTrue( httpParameters.checkParameters()); Assert.assertEquals(10000,httpParameters.getConnectTimeout() ); From 3eb367096a4b092a22c086e208bebd0e2dd1767a Mon Sep 17 00:00:00 2001 From: simon824 Date: Sat, 13 Jun 2020 09:52:58 +0800 Subject: [PATCH 38/38] Resolved the conflict --- .../server/worker/task/http/HttpTaskTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTaskTest.java index e50a54a372..f0d5d79d00 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTaskTest.java @@ -24,6 +24,7 @@ import java.util.Date; import org.apache.dolphinscheduler.common.enums.HttpCheckCondition; import org.apache.dolphinscheduler.common.enums.HttpMethod; import org.apache.dolphinscheduler.common.task.http.HttpParameters; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; @@ -47,7 +48,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; -import com.alibaba.fastjson.JSON; @RunWith(PowerMockRunner.class) @PrepareForTest(OSUtils.class) @@ -123,7 +123,7 @@ public class HttpTaskTest { public void testGenerator(){ String paramJson = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://github.com/\"," + "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}"; - HttpParameters httpParameters = JSON.parseObject(paramJson, HttpParameters.class); + HttpParameters httpParameters = JSONUtils.parseObject(paramJson, HttpParameters.class); Assert.assertEquals(10000,httpParameters.getConnectTimeout() );