github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Using Jackson instead of Fastjson (#2850) 

* Using Jackson instead of Fastjson

* Fix some json bugs

Co-authored-by: dailidong <dailidong66@gmail.com>
pull/2/head
Han Gao 5 years ago committed by GitHub
parent
c8ab27acb0
commit
8f8ddae030
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 589 additions and 460 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java
  2. 105
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java
  3. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java
  4. 21
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/DataSourceParam.java
  5. 40
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParam.java
  6. 6
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/ProcessAddTaskParam.java
  7. 8
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java
  8. 19
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DataSourceParamTest.java
  9. 34
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParamTest.java
  10. 42
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java
  11. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
  12. 36
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StreamUtils.java
  13. 36
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java
  14. 8
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java
  15. 39
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StreamUtilsTest.java

2
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java
Unescape View File

@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.alert.utils;
import com.fasterxml.jackson.databind.JsonNode;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
@ -109,4 +110,5 @@ public class JSONUtilsTest {
}
}

105
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java
Unescape View File

@ -17,11 +17,12 @@
package org.apache.dolphinscheduler.api.service;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.dolphinscheduler.api.dto.ProcessMeta;
import org.apache.dolphinscheduler.api.dto.treeview.Instance;
import org.apache.dolphinscheduler.api.dto.treeview.TreeViewDto;
@ -73,6 +74,11 @@ public class ProcessDefinitionService extends BaseDAGService {
private static final Logger logger = LoggerFactory.getLogger(ProcessDefinitionService.class);
private static final String PROCESSDEFINITIONID = "processDefinitionId";
private static final String RELEASESTATE = "releaseState";
private static final String TASKS = "tasks";
@Autowired
private ProjectMapper projectMapper;
@ -169,6 +175,7 @@ public class ProcessDefinitionService extends BaseDAGService {
/**
* get resource ids
*
* @param processData process data
* @return resource ids
*/
@ -513,7 +520,7 @@ public class ProcessDefinitionService extends BaseDAGService {
// check state
if (null == state) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "releaseState");
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, RELEASESTATE);
return result;
}
@ -525,12 +532,12 @@ public class ProcessDefinitionService extends BaseDAGService {
String resourceIds = processDefinition.getResourceIds();
if (StringUtils.isNotBlank(resourceIds)) {
Integer[] resourceIdArray = Arrays.stream(resourceIds.split(",")).map(Integer::parseInt).toArray(Integer[]::new);
PermissionCheck<Integer> permissionCheck = new PermissionCheck(AuthorizationType.RESOURCE_FILE_ID,processService,resourceIdArray,loginUser.getId(),logger);
PermissionCheck<Integer> permissionCheck = new PermissionCheck<>(AuthorizationType.RESOURCE_FILE_ID, processService, resourceIdArray, loginUser.getId(), logger);
try {
permissionCheck.checkPermission();
} catch (Exception e) {
logger.error(e.getMessage(), e);
putMsg(result, Status.RESOURCE_NOT_EXIST_OR_NO_PERMISSION, "releaseState");
putMsg(result, Status.RESOURCE_NOT_EXIST_OR_NO_PERMISSION, RELEASESTATE);
return result;
}
}
@ -554,7 +561,7 @@ public class ProcessDefinitionService extends BaseDAGService {
}
break;
default:
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "releaseState");
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, RELEASESTATE);
return result;
}
@ -564,6 +571,7 @@ public class ProcessDefinitionService extends BaseDAGService {
/**
* batch export process definition by ids
*
* @param loginUser
* @param projectName
* @param processDefinitionIds
@ -596,6 +604,7 @@ public class ProcessDefinitionService extends BaseDAGService {
/**
* get process definition list by ids
*
* @param processDefinitionIds
* @return
*/
@ -616,6 +625,7 @@ public class ProcessDefinitionService extends BaseDAGService {
/**
* download the process definition file
*
* @param response
* @param processDefinitionList
*/
@ -651,6 +661,7 @@ public class ProcessDefinitionService extends BaseDAGService {
/**
* get export process metadata string
*
* @param processDefinitionId process definition id
* @param processDefinition process definition
* @return export process metadata string
@ -662,6 +673,7 @@ public class ProcessDefinitionService extends BaseDAGService {
/**
* get export process metadata string
*
* @param processDefinitionId process definition id
* @param processDefinition process definition
* @return export process metadata string
@ -699,17 +711,18 @@ public class ProcessDefinitionService extends BaseDAGService {
/**
* correct task param which has datasource or dependent
*
* @param processDefinitionJson processDefinitionJson
* @return correct processDefinitionJson
*/
public String addExportTaskNodeSpecialParam(String processDefinitionJson) {
JSONObject jsonObject = JSONUtils.parseObject(processDefinitionJson);
JSONArray jsonArray = (JSONArray) jsonObject.get("tasks");
ObjectNode jsonObject = JSONUtils.parseObject(processDefinitionJson);
ArrayNode jsonArray = (ArrayNode) jsonObject.path(TASKS);
for (int i = 0; i < jsonArray.size(); i++) {
JSONObject taskNode = jsonArray.getJSONObject(i);
if (StringUtils.isNotEmpty(taskNode.getString("type"))) {
String taskType = taskNode.getString("type");
JsonNode taskNode = jsonArray.path(i);
if (StringUtils.isNotEmpty(taskNode.path("type").asText())) {
String taskType = taskNode.path("type").asText();
ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType);
if (null != addTaskParam) {
@ -717,12 +730,13 @@ public class ProcessDefinitionService extends BaseDAGService {
}
}
}
jsonObject.put("tasks", jsonArray);
jsonObject.set(TASKS, jsonArray);
return jsonObject.toString();
}
/**
* check task if has sub process
*
* @param taskType task type
* @return if task has sub process return true else false
*/
@ -732,6 +746,7 @@ public class ProcessDefinitionService extends BaseDAGService {
/**
* import process definition
*
* @param loginUser login user
* @param file process metadata json file
* @param currentProjectName current project name
@ -761,6 +776,7 @@ public class ProcessDefinitionService extends BaseDAGService {
/**
* check and import process definition
*
* @param loginUser
* @param currentProjectName
* @param result
@ -807,8 +823,8 @@ public class ProcessDefinitionService extends BaseDAGService {
//create process definition
Integer processDefinitionId =
Objects.isNull(createProcessResult.get("processDefinitionId"))?
null:Integer.parseInt(createProcessResult.get("processDefinitionId").toString());
Objects.isNull(createProcessResult.get(PROCESSDEFINITIONID)) ?
null : Integer.parseInt(createProcessResult.get(PROCESSDEFINITIONID).toString());
//scheduler param
return getImportProcessScheduleResult(loginUser,
@ -822,6 +838,7 @@ public class ProcessDefinitionService extends BaseDAGService {
/**
* get create process result
*
* @param loginUser
* @param currentProjectName
* @param result
@ -856,6 +873,7 @@ public class ProcessDefinitionService extends BaseDAGService {
/**
* get import process schedule result
*
* @param loginUser
* @param currentProjectName
* @param result
@ -887,6 +905,7 @@ public class ProcessDefinitionService extends BaseDAGService {
/**
* check importance params
*
* @param processMeta
* @param result
* @return
@ -910,18 +929,19 @@ public class ProcessDefinitionService extends BaseDAGService {
/**
* import process add special task param
*
* @param loginUser login user
* @param processDefinitionJson process definition json
* @param targetProject target project
* @return import process param
*/
private String addImportTaskNodeParam(User loginUser, String processDefinitionJson, Project targetProject) {
JSONObject jsonObject = JSONUtils.parseObject(processDefinitionJson);
JSONArray jsonArray = (JSONArray) jsonObject.get("tasks");
ObjectNode jsonObject = JSONUtils.parseObject(processDefinitionJson);
ArrayNode jsonArray = (ArrayNode) jsonObject.get(TASKS);
//add sql and dependent param
for (int i = 0; i < jsonArray.size(); i++) {
JSONObject taskNode = jsonArray.getJSONObject(i);
String taskType = taskNode.getString("type");
JsonNode taskNode = jsonArray.path(i);
String taskType = taskNode.path("type").asText();
ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType);
if (null != addTaskParam) {
addTaskParam.addImportSpecialParam(taskNode);
@ -931,20 +951,21 @@ public class ProcessDefinitionService extends BaseDAGService {
//recursive sub-process parameter correction map key for old process id value for new process id
Map<Integer, Integer> subProcessIdMap = new HashMap<>(20);
List<Object> subProcessList = jsonArray.stream()
.filter(elem -> checkTaskHasSubProcess(JSONUtils.parseObject(elem.toString()).getString("type")))
List<Object> subProcessList = StreamUtils.asStream(jsonArray.elements())
.filter(elem -> checkTaskHasSubProcess(JSONUtils.parseObject(elem.toString()).path("type").asText()))
.collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(subProcessList)) {
importSubProcess(loginUser, targetProject, jsonArray, subProcessIdMap);
}
jsonObject.put("tasks", jsonArray);
jsonObject.set(TASKS, jsonArray);
return jsonObject.toString();
}
/**
* import process schedule
*
* @param loginUser login user
* @param currentProjectName current project name
* @param processMeta process meta data
@ -998,32 +1019,37 @@ public class ProcessDefinitionService extends BaseDAGService {
/**
* check import process has sub process
* recursion create sub process
*
* @param loginUser login user
* @param targetProject target project
* @param jsonArray process task array
* @param subProcessIdMap correct sub process id map
*/
public void importSubProcess(User loginUser, Project targetProject, JSONArray jsonArray, Map<Integer, Integer> subProcessIdMap) {
public void importSubProcess(User loginUser, Project targetProject, ArrayNode jsonArray, Map<Integer, Integer> subProcessIdMap) {
for (int i = 0; i < jsonArray.size(); i++) {
JSONObject taskNode = jsonArray.getJSONObject(i);
String taskType = taskNode.getString("type");
ObjectNode taskNode = (ObjectNode) jsonArray.path(i);
String taskType = taskNode.path("type").asText();
if (checkTaskHasSubProcess(taskType)) {
if (!checkTaskHasSubProcess(taskType)) {
continue;
}
//get sub process info
JSONObject subParams = JSONUtils.parseObject(taskNode.getString("params"));
Integer subProcessId = subParams.getInteger("processDefinitionId");
ObjectNode subParams = (ObjectNode) taskNode.path("params");
Integer subProcessId = subParams.path(PROCESSDEFINITIONID).asInt();
ProcessDefinition subProcess = processDefineMapper.queryByDefineId(subProcessId);
//check is sub process exist in db
if (null != subProcess) {
if (null == subProcess) {
continue;
}
String subProcessJson = subProcess.getProcessDefinitionJson();
//check current project has sub process
ProcessDefinition currentProjectSubProcess = processDefineMapper.queryByDefineName(targetProject.getId(), subProcess.getName());
if (null == currentProjectSubProcess) {
JSONArray subJsonArray = (JSONArray) JSONUtils.parseObject(subProcess.getProcessDefinitionJson()).get("tasks");
ArrayNode subJsonArray = (ArrayNode) JSONUtils.parseObject(subProcess.getProcessDefinitionJson()).get(TASKS);
List<Object> subProcessList = subJsonArray.stream()
.filter(item -> checkTaskHasSubProcess(JSONUtils.parseObject(item.toString()).getString("type")))
List<Object> subProcessList = StreamUtils.asStream(subJsonArray.elements())
.filter(item -> checkTaskHasSubProcess(JSONUtils.parseObject(item.toString()).path("type").asText()))
.collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(subProcessList)) {
@ -1071,10 +1097,8 @@ public class ProcessDefinitionService extends BaseDAGService {
if (null != newSubProcessDefine) {
subProcessIdMap.put(subProcessId, newSubProcessDefine.getId());
subParams.put("processDefinitionId", newSubProcessDefine.getId());
taskNode.put("params", subParams);
}
}
subParams.put(PROCESSDEFINITIONID, newSubProcessDefine.getId());
taskNode.set("params", subParams);
}
}
}
@ -1138,9 +1162,8 @@ public class ProcessDefinitionService extends BaseDAGService {
*
* @param defineId define id
* @return task node list
* @throws Exception exception
*/
public Map<String, Object> getTaskNodeListByDefinitionId(Integer defineId) throws Exception {
public Map<String, Object> getTaskNodeListByDefinitionId(Integer defineId) {
Map<String, Object> result = new HashMap<>();
ProcessDefinition processDefinition = processDefineMapper.selectById(defineId);
@ -1176,9 +1199,8 @@ public class ProcessDefinitionService extends BaseDAGService {
*
* @param defineIdList define id list
* @return task node list
* @throws Exception exception
*/
public Map<String, Object> getTaskNodeListByDefinitionIdList(String defineIdList) throws Exception {
public Map<String, Object> getTaskNodeListByDefinitionIdList(String defineIdList) {
Map<String, Object> result = new HashMap<>();
Map<Integer, List<TaskNode>> taskNodeMap = new HashMap<>();
@ -1364,9 +1386,8 @@ public class ProcessDefinitionService extends BaseDAGService {
*
* @param processDefinition process definition
* @return dag graph
* @throws Exception if exception happens
*/
private DAG<String, TaskNode, TaskNodeRelation> genDagGraph(ProcessDefinition processDefinition) throws Exception {
private DAG<String, TaskNode, TaskNodeRelation> genDagGraph(ProcessDefinition processDefinition) {
String processDefinitionJson = processDefinition.getProcessDefinitionJson();
@ -1386,8 +1407,6 @@ public class ProcessDefinitionService extends BaseDAGService {
}
/**
* whether the graph has a ring
*

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java
Unescape View File

@ -504,9 +504,8 @@ public class ProcessInstanceService extends BaseDAGService {
*
* @param processInstanceId process instance id
* @return variables data
* @throws Exception exception
*/
public Map<String, Object> viewVariables( Integer processInstanceId) throws Exception {
public Map<String, Object> viewVariables(Integer processInstanceId) {
Map<String, Object> result = new HashMap<>(5);
ProcessInstance processInstance = processInstanceMapper.queryDetailById(processInstanceId);
@ -537,7 +536,7 @@ public class ProcessInstanceService extends BaseDAGService {
List<TaskNode> taskNodeList = workflowData.getTasks();
// global param string
String globalParamStr = JSON.toJSONString(globalParams);
String globalParamStr = JSONUtils.toJson(globalParams);
globalParamStr = ParameterUtils.convertParameterPlaceholders(globalParamStr, timeParams);
globalParams = JSON.parseArray(globalParamStr, Property.class);
for (Property property : globalParams) {

21
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/DataSourceParam.java
Unescape View File

@ -16,9 +16,9 @@
*/
package org.apache.dolphinscheduler.api.utils.exportprocess;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
import org.springframework.beans.factory.InitializingBean;
@ -33,6 +33,7 @@ import java.util.List;
@Service
public class DataSourceParam implements ProcessAddTaskParam, InitializingBean {
private static final String PARAMS = "params";
@Autowired
private DataSourceMapper dataSourceMapper;
@ -42,14 +43,14 @@ public class DataSourceParam implements ProcessAddTaskParam, InitializingBean {
* @return task node json object
*/
@Override
public JSONObject addExportSpecialParam(JSONObject taskNode) {
public JsonNode addExportSpecialParam(JsonNode taskNode) {
// add sqlParameters
JSONObject sqlParameters = JSONUtils.parseObject(taskNode.getString("params"));
DataSource dataSource = dataSourceMapper.selectById((Integer) sqlParameters.get("datasource"));
ObjectNode sqlParameters = (ObjectNode) taskNode.path(PARAMS);
DataSource dataSource = dataSourceMapper.selectById(sqlParameters.get("datasource").asInt());
if (null != dataSource) {
sqlParameters.put("datasourceName", dataSource.getName());
}
taskNode.put("params", sqlParameters);
((ObjectNode)taskNode).set(PARAMS, sqlParameters);
return taskNode;
}
@ -60,14 +61,14 @@ public class DataSourceParam implements ProcessAddTaskParam, InitializingBean {
* @return task node json object
*/
@Override
public JSONObject addImportSpecialParam(JSONObject taskNode) {
JSONObject sqlParameters = JSONUtils.parseObject(taskNode.getString("params"));
List<DataSource> dataSources = dataSourceMapper.queryDataSourceByName(sqlParameters.getString("datasourceName"));
public JsonNode addImportSpecialParam(JsonNode taskNode) {
ObjectNode sqlParameters = (ObjectNode) taskNode.path(PARAMS);
List<DataSource> dataSources = dataSourceMapper.queryDataSourceByName(sqlParameters.path("datasourceName").asText());
if (!dataSources.isEmpty()) {
DataSource dataSource = dataSources.get(0);
sqlParameters.put("datasource", dataSource.getId());
}
taskNode.put("params", sqlParameters);
((ObjectNode)taskNode).set(PARAMS, sqlParameters);
return taskNode;
}

40
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParam.java
Unescape View File

@ -16,8 +16,9 @@
*/
package org.apache.dolphinscheduler.api.utils.exportprocess;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
@ -34,6 +35,7 @@ import org.springframework.stereotype.Service;
@Service
public class DependentParam implements ProcessAddTaskParam, InitializingBean {
private static final String DEPENDENCE = "dependence";
@Autowired
ProcessDefinitionMapper processDefineMapper;
@ -47,18 +49,18 @@ public class DependentParam implements ProcessAddTaskParam, InitializingBean {
* @return task node json object
*/
@Override
public JSONObject addExportSpecialParam(JSONObject taskNode) {
public JsonNode addExportSpecialParam(JsonNode taskNode) {
// add dependent param
JSONObject dependentParameters = JSONUtils.parseObject(taskNode.getString("dependence"));
ObjectNode dependentParameters = JSONUtils.parseObject(taskNode.path(DEPENDENCE).asText());
if (null != dependentParameters) {
JSONArray dependTaskList = (JSONArray) dependentParameters.get("dependTaskList");
ArrayNode dependTaskList = (ArrayNode) dependentParameters.get("dependTaskList");
for (int j = 0; j < dependTaskList.size(); j++) {
JSONObject dependentTaskModel = dependTaskList.getJSONObject(j);
JSONArray dependItemList = (JSONArray) dependentTaskModel.get("dependItemList");
JsonNode dependentTaskModel = dependTaskList.path(j);
ArrayNode dependItemList = (ArrayNode) dependentTaskModel.get("dependItemList");
for (int k = 0; k < dependItemList.size(); k++) {
JSONObject dependentItem = dependItemList.getJSONObject(k);
int definitionId = dependentItem.getInteger("definitionId");
ObjectNode dependentItem = (ObjectNode) dependItemList.path(k);
int definitionId = dependentItem.path("definitionId").asInt();
ProcessDefinition definition = processDefineMapper.queryByDefineId(definitionId);
if (null != definition) {
dependentItem.put("projectName", definition.getProjectName());
@ -66,7 +68,7 @@ public class DependentParam implements ProcessAddTaskParam, InitializingBean {
}
}
}
taskNode.put("dependence", dependentParameters);
((ObjectNode)taskNode).set(DEPENDENCE, dependentParameters);
}
return taskNode;
@ -78,18 +80,18 @@ public class DependentParam implements ProcessAddTaskParam, InitializingBean {
* @return
*/
@Override
public JSONObject addImportSpecialParam(JSONObject taskNode) {
JSONObject dependentParameters = JSONUtils.parseObject(taskNode.getString("dependence"));
public JsonNode addImportSpecialParam(JsonNode taskNode) {
ObjectNode dependentParameters = JSONUtils.parseObject(taskNode.path(DEPENDENCE).asText());
if(dependentParameters != null){
JSONArray dependTaskList = (JSONArray) dependentParameters.get("dependTaskList");
ArrayNode dependTaskList = (ArrayNode) dependentParameters.path("dependTaskList");
for (int h = 0; h < dependTaskList.size(); h++) {
JSONObject dependentTaskModel = dependTaskList.getJSONObject(h);
JSONArray dependItemList = (JSONArray) dependentTaskModel.get("dependItemList");
ObjectNode dependentTaskModel = (ObjectNode) dependTaskList.path(h);
ArrayNode dependItemList = (ArrayNode) dependentTaskModel.get("dependItemList");
for (int k = 0; k < dependItemList.size(); k++) {
JSONObject dependentItem = dependItemList.getJSONObject(k);
Project dependentItemProject = projectMapper.queryByName(dependentItem.getString("projectName"));
ObjectNode dependentItem = (ObjectNode) dependItemList.path(k);
Project dependentItemProject = projectMapper.queryByName(dependentItem.path("projectName").asText());
if(dependentItemProject != null){
ProcessDefinition definition = processDefineMapper.queryByDefineName(dependentItemProject.getId(),dependentItem.getString("definitionName"));
ProcessDefinition definition = processDefineMapper.queryByDefineName(dependentItemProject.getId(),dependentItem.path("definitionName").asText());
if(definition != null){
dependentItem.put("projectId",dependentItemProject.getId());
dependentItem.put("definitionId",definition.getId());
@ -97,7 +99,7 @@ public class DependentParam implements ProcessAddTaskParam, InitializingBean {
}
}
}
taskNode.put("dependence", dependentParameters);
((ObjectNode)taskNode).set(DEPENDENCE, dependentParameters);
}
return taskNode;
}

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/ProcessAddTaskParam.java
Unescape View File

@ -16,7 +16,7 @@
*/
package org.apache.dolphinscheduler.api.utils.exportprocess;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
/**
* ProcessAddTaskParam
@ -28,12 +28,12 @@ public interface ProcessAddTaskParam {
* @param taskNode task node json object
* @return task node json object
*/
JSONObject addExportSpecialParam(JSONObject taskNode);
JsonNode addExportSpecialParam(JsonNode taskNode);
/**
* add task special param: sql task dependent task
* @param taskNode task node json object
* @return task node json object
*/
JSONObject addImportSpecialParam(JSONObject taskNode);
JsonNode addImportSpecialParam(JsonNode taskNode);
}

8
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java
Unescape View File

@ -16,8 +16,8 @@
*/
package org.apache.dolphinscheduler.api.service;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.dolphinscheduler.api.ApiApplicationServer;
import org.apache.dolphinscheduler.api.dto.ProcessMeta;
import org.apache.dolphinscheduler.api.enums.Status;
@ -621,8 +621,8 @@ public class ProcessDefinitionServiceTest {
"\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}";
JSONObject jsonObject = JSONUtils.parseObject(topProcessJson);
JSONArray jsonArray = (JSONArray) jsonObject.get("tasks");
ObjectNode jsonObject = JSONUtils.parseObject(topProcessJson);
ArrayNode jsonArray = (ArrayNode) jsonObject.path("tasks");
String originSubJson = jsonArray.toString();

19
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DataSourceParamTest.java
Unescape View File

@ -16,7 +16,8 @@
*/
package org.apache.dolphinscheduler.api.utils.exportprocess;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.dolphinscheduler.api.ApiApplicationServer;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
@ -48,13 +49,13 @@ public class DataSourceParamTest {
"\"preTasks\":[\"dependent\"]}";
JSONObject taskNode = JSONUtils.parseObject(sqlJson);
if (StringUtils.isNotEmpty(taskNode.getString("type"))) {
String taskType = taskNode.getString("type");
ObjectNode taskNode = JSONUtils.parseObject(sqlJson);
if (StringUtils.isNotEmpty(taskNode.path("type").asText())) {
String taskType = taskNode.path("type").asText();
ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType);
JSONObject sql = addTaskParam.addExportSpecialParam(taskNode);
JsonNode sql = addTaskParam.addExportSpecialParam(taskNode);
JSONAssert.assertEquals(taskNode.toString(), sql.toString(), false);
}
@ -72,13 +73,13 @@ public class DataSourceParamTest {
"\"taskInstancePriority\":\"MEDIUM\",\"name\":\"mysql\",\"dependence\":{}," +
"\"retryInterval\":\"1\",\"preTasks\":[\"dependent\"],\"id\":\"tasks-8745\"}";
JSONObject taskNode = JSONUtils.parseObject(sqlJson);
if (StringUtils.isNotEmpty(taskNode.getString("type"))) {
String taskType = taskNode.getString("type");
ObjectNode taskNode = JSONUtils.parseObject(sqlJson);
if (StringUtils.isNotEmpty(taskNode.path("type").asText())) {
String taskType = taskNode.path("type").asText();
ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType);
JSONObject sql = addTaskParam.addImportSpecialParam(taskNode);
JsonNode sql = addTaskParam.addImportSpecialParam(taskNode);
JSONAssert.assertEquals(taskNode.toString(), sql.toString(), false);
}

34
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParamTest.java
Unescape View File

@ -17,6 +17,8 @@
package org.apache.dolphinscheduler.api.utils.exportprocess;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.dolphinscheduler.api.ApiApplicationServer;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
@ -43,13 +45,13 @@ public class DependentParamTest {
"\"dependItemList\":[{\"projectId\":2,\"definitionId\":46,\"depTasks\":\"ALL\"," +
"\"cycle\":\"day\",\"dateValue\":\"today\"}]}]}}";
JSONObject taskNode = JSONUtils.parseObject(dependentJson);
if (StringUtils.isNotEmpty(taskNode.getString("type"))) {
String taskType = taskNode.getString("type");
ObjectNode taskNode = JSONUtils.parseObject(dependentJson);
if (StringUtils.isNotEmpty(taskNode.path("type").asText())) {
String taskType = taskNode.path("type").asText();
ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType);
JSONObject dependent = addTaskParam.addExportSpecialParam(taskNode);
JsonNode dependent = addTaskParam.addExportSpecialParam(taskNode);
JSONAssert.assertEquals(taskNode.toString(), dependent.toString(), false);
}
@ -57,13 +59,13 @@ public class DependentParamTest {
String dependentEmpty = "{\"type\":\"DEPENDENT\",\"id\":\"tasks-33787\"," +
"\"name\":\"dependent\",\"params\":{},\"description\":\"\",\"runFlag\":\"NORMAL\"}";
JSONObject taskEmpty = JSONUtils.parseObject(dependentEmpty);
if (StringUtils.isNotEmpty(taskEmpty.getString("type"))) {
String taskType = taskEmpty.getString("type");
ObjectNode taskEmpty = JSONUtils.parseObject(dependentEmpty);
if (StringUtils.isNotEmpty(taskEmpty.path("type").asText())) {
String taskType = taskEmpty.path("type").asText();
ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType);
JSONObject dependent = addTaskParam.addImportSpecialParam(taskEmpty);
JsonNode dependent = addTaskParam.addImportSpecialParam(taskEmpty);
JSONAssert.assertEquals(taskEmpty.toString(), dependent.toString(), false);
}
@ -81,13 +83,13 @@ public class DependentParamTest {
"\"projectId\":1,\"cycle\":\"day\",\"definitionId\":7}],\"relation\":\"AND\"}]," +
"\"relation\":\"AND\"},\"retryInterval\":\"1\",\"preTasks\":[],\"id\":\"tasks-55485\"}";
JSONObject taskNode = JSONUtils.parseObject(dependentJson);
if (StringUtils.isNotEmpty(taskNode.getString("type"))) {
String taskType = taskNode.getString("type");
ObjectNode taskNode = JSONUtils.parseObject(dependentJson);
if (StringUtils.isNotEmpty(taskNode.path("type").asText())) {
String taskType = taskNode.path("type").asText();
ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType);
JSONObject dependent = addTaskParam.addImportSpecialParam(taskNode);
JsonNode dependent = addTaskParam.addImportSpecialParam(taskNode);
JSONAssert.assertEquals(taskNode.toString(), dependent.toString(), false);
}
@ -97,13 +99,13 @@ public class DependentParamTest {
"\"strategy\":\"\"},\"maxRetryTimes\":\"0\",\"taskInstancePriority\":\"MEDIUM\"" +
",\"name\":\"dependent\",\"retryInterval\":\"1\",\"preTasks\":[],\"id\":\"tasks-55485\"}";
JSONObject taskNodeEmpty = JSONUtils.parseObject(dependentEmpty);
if (StringUtils.isNotEmpty(taskNodeEmpty.getString("type"))) {
String taskType = taskNodeEmpty.getString("type");
JsonNode taskNodeEmpty = JSONUtils.parseObject(dependentEmpty);
if (StringUtils.isNotEmpty(taskNodeEmpty.path("type").asText())) {
String taskType = taskNodeEmpty.path("type").asText();
ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType);
JSONObject dependent = addTaskParam.addImportSpecialParam(taskNode);
JsonNode dependent = addTaskParam.addImportSpecialParam(taskNode);
JSONAssert.assertEquals(taskNodeEmpty.toString(), dependent.toString(), false);
}

42
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java
Unescape View File

@ -16,13 +16,12 @@
*/
package org.apache.dolphinscheduler.common.utils;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.TypeReference;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.*;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -43,18 +42,23 @@ public class JSONUtils {
private static final ObjectMapper objectMapper = new ObjectMapper();
private JSONUtils() {
}
static {
//Feature that determines whether encountering of unknown properties, false means not analyzer unknown properties
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false).setTimeZone(TimeZone.getDefault());
objectMapper.configure(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true).setTimeZone(TimeZone.getDefault());
}
/**
* json representation of object
*
* @param object object
* @return object to json string
*/
public static String toJson(Object object) {
try {
return JSON.toJSONString(object,false);
return objectMapper.writeValueAsString(object);
} catch (Exception e) {
logger.error("object to json exception!", e);
}
@ -64,7 +68,6 @@ public class JSONUtils {
/**
*
* This method deserializes the specified Json into an object of the specified class. It is not
* suitable to use if the specified class is a generic type since it will not have the generic
* type information because of the Type Erasure feature of Java. Therefore, this method should not
@ -84,7 +87,7 @@ public class JSONUtils {
}
try {
return JSON.parseObject(json, clazz);
return objectMapper.readValue(json, clazz);
} catch (Exception e) {
logger.error("parse object exception!", e);
}
@ -105,7 +108,8 @@ public class JSONUtils {
return new ArrayList<>();
}
try {
return JSONArray.parseArray(json, clazz);
return objectMapper.readValue(json, new TypeReference<List<T>>() {
});
} catch (Exception e) {
logger.error("JSONArray.parseArray exception!", e);
}
@ -114,7 +118,6 @@ public class JSONUtils {
}
/**
* check json object valid
*
@ -145,7 +148,6 @@ public class JSONUtils {
*
* @param jsonNode json node
* @param fieldName Name of field to look for
*
* @return Value of first matching node found, if any; null if none
*/
public static String findValue(JsonNode jsonNode, String fieldName) {
@ -161,7 +163,7 @@ public class JSONUtils {
/**
* json to map
*
* <p>
* {@link #toMap(String, Class, Class)}
*
* @param json json
@ -173,7 +175,7 @@ public class JSONUtils {
}
try {
return JSON.parseObject(json, new TypeReference<HashMap<String, String>>(){});
return objectMapper.readValue(json, new TypeReference<Map<String, String>>() {});
} catch (Exception e) {
logger.error("json to map exception!", e);
}
@ -182,7 +184,6 @@ public class JSONUtils {
}
/**
*
* json to map
*
* @param json json
@ -198,7 +199,8 @@ public class JSONUtils {
}
try {
return JSON.parseObject(json, new TypeReference<HashMap<K, V>>() {});
return objectMapper.readValue(json, new TypeReference<Map<K, V>>() {
});
} catch (Exception e) {
logger.error("json to map exception!", e);
}
@ -208,35 +210,35 @@ public class JSONUtils {
/**
* object to json string
*
* @param object object
* @return json string
*/
public static String toJsonString(Object object) {
try {
return JSON.toJSONString(object,false);
return objectMapper.writeValueAsString(object);
} catch (Exception e) {
throw new RuntimeException("Object json deserialization exception.", e);
}
}
public static JSONObject parseObject(String text) {
public static ObjectNode parseObject(String text) {
try {
return JSON.parseObject(text);
return (ObjectNode) objectMapper.readTree(text);
} catch (Exception e) {
throw new RuntimeException("String json deserialization exception.", e);
}
}
public static JSONArray parseArray(String text) {
public static ArrayNode parseArray(String text) {
try {
return JSON.parseArray(text);
return (ArrayNode) objectMapper.readTree(text);
} catch (Exception e) {
throw new RuntimeException("Json deserialization exception.", e);
}
}
/**
* json serializer
*/

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
Unescape View File

@ -196,7 +196,7 @@ public class ParameterUtils {
property.setValue(val);
}
}
return JSON.toJSONString(globalParamList);
return JSONUtils.toJson(globalParamList);
}

36
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StreamUtils.java
Unescape View File

@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import java.util.Iterator;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
public class StreamUtils {
private StreamUtils() { }
public static <T> Stream<T> asStream(Iterator<T> sourceIterator) {
return asStream(sourceIterator, false);
}
public static <T> Stream<T> asStream(Iterator<T> sourceIterator, boolean parallel) {
Iterable<T> iterable = () -> sourceIterator;
return StreamSupport.stream(iterable.spliterator(), parallel);
}
}

36
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java
Unescape View File

@ -17,6 +17,8 @@
package org.apache.dolphinscheduler.common.utils;
import com.alibaba.fastjson.JSON;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.dolphinscheduler.common.enums.DataType;
import org.apache.dolphinscheduler.common.enums.Direct;
import org.apache.dolphinscheduler.common.process.Property;
@ -101,9 +103,6 @@ public class JSONUtilsTest {
@Test
public void testParseObject() {
Assert.assertEquals("{\"foo\":\"bar\"}", JSONUtils.parseObject(
"{\n" + "\"foo\": \"bar\",\n" + "}", String.class));
Assert.assertNull(JSONUtils.parseObject("", null));
Assert.assertNull(JSONUtils.parseObject("foo", String.class));
}
@ -134,15 +133,19 @@ public class JSONUtilsTest {
map.put("foo","bar");
Assert.assertTrue(map.equals(JSONUtils.toMap(
"{\n" + "\"foo\": \"bar\",\n" + "}")));
"{\n" + "\"foo\": \"bar\"\n" + "}")));
Assert.assertFalse(map.equals(JSONUtils.toMap(
"{\n" + "\"bar\": \"foo\",\n" + "}")));
"{\n" + "\"bar\": \"foo\"\n" + "}")));
Assert.assertNull(JSONUtils.toMap("3"));
Assert.assertNull(JSONUtils.toMap(null));
Assert.assertNull(JSONUtils.toMap("3", null, null));
Assert.assertNull(JSONUtils.toMap(null, null, null));
String str = "{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-1\\\"\"}";
Map<String, String> m = JSONUtils.toMap(str);
Assert.assertNotNull(m);
}
@Test
@ -155,4 +158,27 @@ public class JSONUtilsTest {
Assert.assertEquals(String.valueOf((Object) null),
JSONUtils.toJsonString(null));
}
@Test
public void parseObject() {
String str = "{\"color\":\"yellow\",\"type\":\"renault\"}";
ObjectNode node = JSONUtils.parseObject(str);
Assert.assertEquals("yellow", node.path("color").asText());
node.put("price", 100);
Assert.assertEquals(100, node.path("price").asInt());
node.put("color", "red");
Assert.assertEquals("red", node.path("color").asText());
}
@Test
public void parseArray() {
String str = "[{\"color\":\"yellow\",\"type\":\"renault\"}]";
ArrayNode node = JSONUtils.parseArray(str);
Assert.assertEquals("yellow", node.path(0).path("color").asText());
}
}

8
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java
Unescape View File

@ -91,13 +91,13 @@ public class ParameterUtilsTest {
globalParamList.add(property);
String result2 = ParameterUtils.curingGlobalParams(null,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,scheduleTime);
Assert.assertEquals(result2, JSON.toJSONString(globalParamList));
Assert.assertEquals(result2, JSONUtils.toJson(globalParamList));
String result3 = ParameterUtils.curingGlobalParams(globalParamMap,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,null);
Assert.assertEquals(result3, JSON.toJSONString(globalParamList));
Assert.assertEquals(result3, JSONUtils.toJson(globalParamList));
String result4 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime);
Assert.assertEquals(result4, JSON.toJSONString(globalParamList));
Assert.assertEquals(result4, JSONUtils.toJson(globalParamList));
//test var $ startsWith
globalParamMap.put("bizDate","${system.biz.date}");
@ -113,7 +113,7 @@ public class ParameterUtilsTest {
globalParamList.add(property4);
String result5 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime);
Assert.assertEquals(result5,JSONUtils.toJsonString(globalParamList));
Assert.assertEquals(result5, JSONUtils.toJson(globalParamList));
}
/**

39
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StreamUtilsTest.java
Unescape View File

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.junit.Assert;
import org.junit.Test;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import static org.junit.Assert.*;
public class StreamUtilsTest {
@Test
public void asStream() {
List<String> list = Arrays.asList("a", "b", "c");
List<String> ret = StreamUtils.asStream(list.iterator())
.filter(item -> item.equals("a"))
.collect(Collectors.toList());
Assert.assertEquals("a", ret.get(0));
}
}
Write Preview
Loading…
Cancel
Save