diff --git a/dolphinscheduler-alert/pom.xml b/dolphinscheduler-alert/pom.xml index 188bfa3d9e..d63e3808b4 100644 --- a/dolphinscheduler-alert/pom.xml +++ b/dolphinscheduler-alert/pom.xml @@ -66,11 +66,6 @@ commons-email - - com.alibaba - fastjson - - com.fasterxml.jackson.core jackson-core diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java index 79123a1e80..7331df3913 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java @@ -16,13 +16,15 @@ */ package org.apache.dolphinscheduler.alert.template.impl; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import org.apache.dolphinscheduler.alert.template.AlertTemplate; import org.apache.dolphinscheduler.alert.utils.Constants; -import org.apache.dolphinscheduler.alert.utils.JSONUtils; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.dolphinscheduler.common.utils.*; import java.util.*; @@ -107,18 +109,11 @@ public class DefaultHTMLTemplate implements AlertTemplate { private String getTextTypeMessage(String content,boolean showAll){ if (StringUtils.isNotEmpty(content)){ - List list; - try { - list = JSONUtils.toList(content,String.class); - }catch (Exception e){ - logger.error("json format exception",e); - return null; - } - + ArrayNode list = JSONUtils.parseArray(content); StringBuilder contents = new StringBuilder(100); - for (String str : list){ + for (JsonNode jsonNode : list){ contents.append(Constants.TR); - contents.append(Constants.TD).append(str).append(Constants.TD_END); + contents.append(Constants.TD).append(jsonNode.toString()).append(Constants.TD_END); contents.append(Constants.TR_END); } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java index f2c9cd812c..455d5de834 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.alert.utils; -import com.alibaba.fastjson.JSON; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.commons.codec.binary.StringUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; @@ -129,7 +129,7 @@ public class DingTalkUtils { textContent.put("content", txt); items.put("text", textContent); - return JSON.toJSONString(items); + return JSONUtils.toJsonString(items); } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index d199d154aa..38dc38b4a8 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -18,9 +18,8 @@ package org.apache.dolphinscheduler.alert.utils; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.utils.StringUtils; -import com.alibaba.fastjson.JSON; +import org.apache.dolphinscheduler.common.utils.*; -import com.google.common.reflect.TypeToken; import org.apache.dolphinscheduler.plugin.model.AlertData; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; @@ -64,14 +63,15 @@ public class EnterpriseWeChatUtils { /** * get Enterprise WeChat is enable + * * @return isEnable */ - public static boolean isEnable(){ + public static boolean isEnable() { Boolean isEnable = null; try { isEnable = PropertyUtils.getBoolean(Constants.ENTERPRISE_WECHAT_ENABLE); } catch (Exception e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); } if (isEnable == null) { return false; @@ -81,6 +81,7 @@ public class EnterpriseWeChatUtils { /** * get Enterprise WeChat token info + * * @return token string info * @throws IOException the IOException */ @@ -99,10 +100,12 @@ public class EnterpriseWeChatUtils { response.close(); } - Map map = JSON.parseObject(resp, - new TypeToken>() { - }.getType()); - return map.get("access_token").toString(); + Map map = JSONUtils.toMap(resp); + if (map != null) { + return map.get("access_token"); + } else { + return null; + } } finally { httpClient.close(); } @@ -110,9 +113,10 @@ public class EnterpriseWeChatUtils { /** * make team single Enterprise WeChat message + * * @param toParty the toParty * @param agentId the agentId - * @param msg the msg + * @param msg the msg * @return Enterprise WeChat send message */ public static String makeTeamSendMsg(String toParty, String agentId, String msg) { @@ -123,9 +127,10 @@ public class EnterpriseWeChatUtils { /** * make team multi Enterprise WeChat message + * * @param toParty the toParty * @param agentId the agentId - * @param msg the msg + * @param msg the msg * @return Enterprise WeChat send message */ public static String makeTeamSendMsg(Collection toParty, String agentId, String msg) { @@ -137,9 +142,10 @@ public class EnterpriseWeChatUtils { /** * make team single user message - * @param toUser the toUser + * + * @param toUser the toUser * @param agentId the agentId - * @param msg the msg + * @param msg the msg * @return Enterprise WeChat send message */ public static String makeUserSendMsg(String toUser, String agentId, String msg) { @@ -150,9 +156,10 @@ public class EnterpriseWeChatUtils { /** * make team multi user message - * @param toUser the toUser + * + * @param toUser the toUser * @param agentId the agentId - * @param msg the msg + * @param msg the msg * @return Enterprise WeChat send message */ public static String makeUserSendMsg(Collection toUser, String agentId, String msg) { @@ -164,9 +171,10 @@ public class EnterpriseWeChatUtils { /** * send Enterprise WeChat + * * @param charset the charset - * @param data the data - * @param token the token + * @param data the data + * @param token the token * @return Enterprise WeChat resp, demo: {"errcode":0,"errmsg":"ok","invaliduser":""} * @throws IOException the IOException */ @@ -196,21 +204,22 @@ public class EnterpriseWeChatUtils { /** * convert table to markdown style - * @param title the title + * + * @param title the title * @param content the content * @return markdown table content */ - public static String markdownTable(String title,String content){ + public static String markdownTable(String title, String content) { List mapItemsList = JSONUtils.toList(content, LinkedHashMap.class); StringBuilder contents = new StringBuilder(200); if (null != mapItemsList) { - for (LinkedHashMap mapItems : mapItemsList){ + for (LinkedHashMap mapItems : mapItemsList) { Set> entries = mapItems.entrySet(); Iterator> iterator = entries.iterator(); - StringBuilder t = new StringBuilder(String.format("`%s`%s",title,Constants.MARKDOWN_ENTER)); + StringBuilder t = new StringBuilder(String.format("`%s`%s", title, Constants.MARKDOWN_ENTER)); - while (iterator.hasNext()){ + while (iterator.hasNext()) { Map.Entry entry = iterator.next(); t.append(Constants.MARKDOWN_QUOTE); @@ -225,23 +234,24 @@ public class EnterpriseWeChatUtils { /** * convert text to markdown style - * @param title the title + * + * @param title the title * @param content the content * @return markdown text */ - public static String markdownText(String title,String content){ - if (StringUtils.isNotEmpty(content)){ + public static String markdownText(String title, String content) { + if (StringUtils.isNotEmpty(content)) { List list; try { - list = JSONUtils.toList(content,String.class); - }catch (Exception e){ - logger.error("json format exception",e); + list = JSONUtils.toList(content, String.class); + } catch (Exception e) { + logger.error("json format exception", e); return null; } StringBuilder contents = new StringBuilder(100); - contents.append(String.format("`%s`%n",title)); - for (String str : list){ + contents.append(String.format("`%s`%n", title)); + for (String str : list) { contents.append(Constants.MARKDOWN_QUOTE); contents.append(str); contents.append(Constants.MARKDOWN_ENTER); @@ -255,14 +265,15 @@ public class EnterpriseWeChatUtils { /** * Determine the mardown style based on the show type of the alert + * * @return the markdown alert table/text */ - public static String markdownByAlert(AlertData alert){ + public static String markdownByAlert(AlertData alert) { String result = ""; if (alert.getShowType().equals(ShowType.TABLE.getDescp())) { - result = markdownTable(alert.getTitle(),alert.getContent()); - }else if(alert.getShowType().equals(ShowType.TEXT.getDescp())){ - result = markdownText(alert.getTitle(),alert.getContent()); + result = markdownTable(alert.getTitle(), alert.getContent()); + } else if (alert.getShowType().equals(ShowType.TEXT.getDescp())) { + result = markdownText(alert.getTitle(), alert.getContent()); } return result; diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java index 366e2828c5..08256860e2 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java @@ -30,6 +30,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.util.*; +import org.apache.dolphinscheduler.common.utils.*; /** * excel utils diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java deleted file mode 100644 index 5f8371de2d..0000000000 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.alert.utils; - -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONArray; -import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collections; -import java.util.List; - -/** - * json utils - */ -public class JSONUtils { - - private static final Logger logger = LoggerFactory.getLogger(JSONUtils.class); - - /** - * object to json string - * @param object the object to be converted to json - * @return json string - */ - public static String toJsonString(Object object) { - try{ - return JSON.toJSONString(object,false); - } catch (Exception e) { - throw new RuntimeException("Json deserialization exception.", e); - } - } - - /** - * json to list - * - * @param json the json - * @param clazz c - * @param the generic clazz - * @return the result list or empty list - */ - public static List toList(String json, Class clazz) { - if (StringUtils.isEmpty(json)) { - return Collections.emptyList(); - } - try { - return JSON.parseArray(json, clazz); - } catch (Exception e) { - logger.error("JSONArray.parseArray exception!",e); - } - - return Collections.emptyList(); - } - -} diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplateTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplateTest.java index 58609c07cb..76d01c134d 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplateTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplateTest.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.alert.template.impl; -import org.apache.dolphinscheduler.alert.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.enums.ShowType; import org.junit.Test; import org.slf4j.Logger; @@ -116,7 +116,7 @@ public class DefaultHTMLTemplateTest{ " \n" + " \n" + " \n" + - "
{\"mysql service name\":\"mysql200\",\"mysql address\":\"192.168.xx.xx\",\"database client connections\":\"190\",\"port\":\"3306\",\"no index of number\":\"80\"}
{\"mysql service name\":\"mysql210\",\"mysql address\":\"192.168.xx.xx\",\"database client connections\":\"90\",\"port\":\"3306\",\"no index of number\":\"10\"}
\n" + + "{\"mysql service name\":\"mysql200\",\"mysql address\":\"192.168.xx.xx\",\"port\":\"3306\",\"no index of number\":\"80\",\"database client connections\":\"190\"}{\"mysql service name\":\"mysql210\",\"mysql address\":\"192.168.xx.xx\",\"port\":\"3306\",\"no index of number\":\"10\",\"database client connections\":\"90\"} \n" + " \n" + ""; } diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java index 2149858361..049881c087 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java @@ -16,13 +16,10 @@ */ package org.apache.dolphinscheduler.alert.utils; -import com.alibaba.fastjson.JSON; import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpPost; import org.apache.http.impl.client.CloseableHttpClient; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mockito; @@ -67,7 +64,7 @@ public class DingTalkUtilsTest { // logger.info(PropertyUtils.getString(Constants.DINGTALK_WEBHOOK)); // String rsp = DingTalkUtils.sendDingTalkMsg(msgTosend, Constants.UTF_8); // logger.info("send msg result:{}",rsp); -// String errmsg = JSON.parseObject(rsp).getString("errmsg"); +// String errmsg = JSONUtils.parseObject(rsp).getString("errmsg"); // Assert.assertEquals("ok", errmsg); // } catch (Exception e) { // e.printStackTrace(); diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java index 2b405cc436..01b81101fd 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java @@ -16,14 +16,12 @@ */ package org.apache.dolphinscheduler.alert.utils; -import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.plugin.model.AlertData; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mockito; @@ -34,6 +32,7 @@ import org.powermock.modules.junit4.PowerMockRunner; import java.io.IOException; import java.util.*; +import org.apache.dolphinscheduler.common.utils.*; /** * Please manually modify the configuration file before testing. @@ -206,7 +205,7 @@ public class EnterpriseWeChatUtilsTest { // String msg = EnterpriseWeChatUtils.makeTeamSendMsg(partyId, agentId, "hello world"); // String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); // -// String errmsg = JSON.parseObject(resp).getString("errmsg"); +// String errmsg = JSONUtils.parseObject(resp).getString("errmsg"); // Assert.assertEquals("ok",errmsg); // } catch (IOException e) { // e.printStackTrace(); @@ -221,7 +220,7 @@ public class EnterpriseWeChatUtilsTest { // String msg = EnterpriseWeChatUtils.makeTeamSendMsg(listPartyId, agentId, "hello world"); // String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); // -// String errmsg = JSON.parseObject(resp).getString("errmsg"); +// String errmsg = JSONUtils.parseObject(resp).getString("errmsg"); // Assert.assertEquals("ok",errmsg); // } catch (IOException e) { // e.printStackTrace(); @@ -248,7 +247,7 @@ public class EnterpriseWeChatUtilsTest { // // String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); // -// String errmsg = JSON.parseObject(resp).getString("errmsg"); +// String errmsg = JSONUtils.parseObject(resp).getString("errmsg"); // Assert.assertEquals("ok",errmsg); // } catch (IOException e) { // e.printStackTrace(); @@ -263,7 +262,7 @@ public class EnterpriseWeChatUtilsTest { // String msg = EnterpriseWeChatUtils.makeUserSendMsg(listUserId, agentId, "hello world"); // String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); // -// String errmsg = JSON.parseObject(resp).getString("errmsg"); +// String errmsg = JSONUtils.parseObject(resp).getString("errmsg"); // Assert.assertEquals("ok",errmsg); // } catch (IOException e) { // e.printStackTrace(); diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java deleted file mode 100644 index 843bcf4083..0000000000 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.alert.utils; - -import com.fasterxml.jackson.databind.JsonNode; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; - -import static org.junit.Assert.*; - -public class JSONUtilsTest { - - private static final Logger logger = LoggerFactory.getLogger(JSONUtilsTest.class); - - public List> list = new ArrayList<>(); - - public String expected = null; - - @Before - public void setUp() throws Exception { - - //Define expected json string - expected = "[{\"mysql service name\":\"mysql200\",\"mysql address\":\"192.168.xx.xx\",\"port\":\"3306\",\"no index of number\":\"80\",\"database client connections\":\"190\"}]"; - - //Initial map - LinkedHashMap map = new LinkedHashMap<>(); - map.put("mysql service name","mysql200"); - map.put("mysql address","192.168.xx.xx"); - map.put("port","3306"); - map.put("no index of number","80"); - map.put("database client connections","190"); - - //Add map into list - list.add(map); - } - - - /** - * Test toJsonString - */ - @Test - public void testToJsonString() { - - //Invoke toJsonString - String result = JSONUtils.toJsonString(list); - logger.info(result); - - //Equal result with expected string - assertEquals(result,expected); - - //If param is null, then return null string - result = JSONUtils.toJsonString(null); - logger.info(result); - - assertEquals("null", result); - - } - - /** - * Test toList - */ - @Test - public void testToList() { - - //Invoke toList - List result = JSONUtils.toList(expected ,LinkedHashMap.class); - //Equal list size=1 - assertEquals(1,result.size()); - - //Transform entity to LinkedHashMap - LinkedHashMap entity = result.get(0); - - //Equal expected values - assertEquals("mysql200",entity.get("mysql service name")); - assertEquals("192.168.xx.xx", entity.get("mysql address")); - assertEquals("3306", entity.get("port")); - assertEquals("80", entity.get("no index of number")); - assertEquals("190", entity.get("database client connections")); - - //If param is null, then return empty list - result = JSONUtils.toList(null ,LinkedHashMap.class); - assertNotNull(result); - assertTrue(result.isEmpty()); - - //If param is incorrect, then return empty list and log error message - result = JSONUtils.toList("}{" ,LinkedHashMap.class); - assertNotNull(result); - assertTrue(result.isEmpty()); - - } - - -} diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java index 11322da0e3..26a69c43ba 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java @@ -29,6 +29,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; +import org.apache.dolphinscheduler.common.utils.*; /** diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index b7c3f3da69..16f611fd66 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -86,11 +86,6 @@ spring-context - - com.alibaba - fastjson - - commons-collections commons-collections diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/ScheduleParam.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/ScheduleParam.java index a842960bc2..a7cdfcfe46 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/ScheduleParam.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/ScheduleParam.java @@ -16,13 +16,17 @@ */ package org.apache.dolphinscheduler.api.dto; +import com.fasterxml.jackson.annotation.JsonFormat; + import java.util.Date; /** * schedule parameters */ public class ScheduleParam { + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date endTime; private String crontab; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/Task.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/Task.java index 55ead2872a..453559e123 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/Task.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/Task.java @@ -16,6 +16,8 @@ */ package org.apache.dolphinscheduler.api.dto.gantt; +import com.fasterxml.jackson.annotation.JsonFormat; + import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -41,16 +43,19 @@ public class Task { /** * task execution date */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date executionDate; /** * task iso start */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date isoStart; /** * task iso end */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date isoEnd; /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java index fb0da702b3..6a5b6e3353 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java @@ -1,7 +1,6 @@ package org.apache.dolphinscheduler.api.dto.resources; -import com.alibaba.fastjson.annotation.JSONField; -import com.alibaba.fastjson.annotation.JSONType; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; import org.apache.dolphinscheduler.common.enums.ResourceType; import java.util.ArrayList; @@ -26,7 +25,7 @@ import java.util.List; /** * resource component */ -@JSONType(orders={"id","pid","name","fullName","description","isDirctory","children","type"}) +@JsonPropertyOrder({"id","pid","name","fullName","description","isDirctory","children","type"}) public abstract class ResourceComponent { public ResourceComponent() { } @@ -46,17 +45,14 @@ public abstract class ResourceComponent { /** * id */ - @JSONField(ordinal = 1) protected int id; /** * parent id */ - @JSONField(ordinal = 2) protected int pid; /** * name */ - @JSONField(ordinal = 3) protected String name; /** * current directory @@ -65,32 +61,26 @@ public abstract class ResourceComponent { /** * full name */ - @JSONField(ordinal = 4) protected String fullName; /** * description */ - @JSONField(ordinal = 5) protected String description; /** * is directory */ - @JSONField(ordinal = 6) protected boolean isDirctory; /** * id value */ - @JSONField(ordinal = 7) protected String idValue; /** * resoruce type */ - @JSONField(ordinal = 8) protected ResourceType type; /** * children */ - @JSONField(ordinal = 8) protected List children = new ArrayList<>(); /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/treeview/Instance.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/treeview/Instance.java index 10bfeb0790..9bab520ed5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/treeview/Instance.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/treeview/Instance.java @@ -16,6 +16,8 @@ */ package org.apache.dolphinscheduler.api.dto.treeview; +import com.fasterxml.jackson.annotation.JsonFormat; + import java.util.Date; /** @@ -42,11 +44,13 @@ public class Instance { /** * node start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** * node end time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date endTime; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java index de2c8d9cea..edc115b3d4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java @@ -20,7 +20,7 @@ import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.ProcessDag; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.ProcessData; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.utils.DagHelper; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index f4e846fbce..1bfacfb93d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -16,11 +16,9 @@ */ package org.apache.dolphinscheduler.api.service; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; -import com.alibaba.fastjson.TypeReference; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; @@ -29,7 +27,7 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbConnectType; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.utils.CommonUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.datasource.*; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.Resource; @@ -161,15 +159,15 @@ public class DataSourceService extends BaseService{ return result; } //check password，if the password is not updated, set to the old password. - JSONObject paramObject = JSON.parseObject(parameter); - String password = paramObject.getString(Constants.PASSWORD); + ObjectNode paramObject = JSONUtils.parseObject(parameter); + String password = paramObject.path(Constants.PASSWORD).asText(); if (StringUtils.isBlank(password)) { String oldConnectionParams = dataSource.getConnectionParams(); - JSONObject oldParams = JSON.parseObject(oldConnectionParams); - paramObject.put(Constants.PASSWORD, oldParams.getString(Constants.PASSWORD)); + ObjectNode oldParams = JSONUtils.parseObject(oldConnectionParams); + paramObject.put(Constants.PASSWORD, oldParams.path(Constants.PASSWORD).asText()); } // connectionParams json - String connectionParams = paramObject.toJSONString(); + String connectionParams = paramObject.toString(); Boolean isConnection = checkConnection(type, connectionParams); if (!isConnection) { @@ -315,9 +313,9 @@ public class DataSourceService extends BaseService{ for (DataSource dataSource : dataSourceList) { String connectionParams = dataSource.getConnectionParams(); - JSONObject object = JSON.parseObject(connectionParams); + ObjectNode object = JSONUtils.parseObject(connectionParams); object.put(Constants.PASSWORD, Constants.XXXXXX); - dataSource.setConnectionParams(JSONUtils.toJson(object)); + dataSource.setConnectionParams(object.toString()); } } @@ -379,11 +377,11 @@ public class DataSourceService extends BaseService{ try { switch (dbType) { case POSTGRESQL: - datasource = JSON.parseObject(parameter, PostgreDataSource.class); + datasource = JSONUtils.parseObject(parameter, PostgreDataSource.class); Class.forName(Constants.ORG_POSTGRESQL_DRIVER); break; case MYSQL: - datasource = JSON.parseObject(parameter, MySQLDataSource.class); + datasource = JSONUtils.parseObject(parameter, MySQLDataSource.class); Class.forName(Constants.COM_MYSQL_JDBC_DRIVER); break; case HIVE: @@ -398,26 +396,26 @@ public class DataSourceService extends BaseService{ getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_PATH)); } if (dbType == DbType.HIVE){ - datasource = JSON.parseObject(parameter, HiveDataSource.class); + datasource = JSONUtils.parseObject(parameter, HiveDataSource.class); }else if (dbType == DbType.SPARK){ - datasource = JSON.parseObject(parameter, SparkDataSource.class); + datasource = JSONUtils.parseObject(parameter, SparkDataSource.class); } Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER); break; case CLICKHOUSE: - datasource = JSON.parseObject(parameter, ClickHouseDataSource.class); + datasource = JSONUtils.parseObject(parameter, ClickHouseDataSource.class); Class.forName(Constants.COM_CLICKHOUSE_JDBC_DRIVER); break; case ORACLE: - datasource = JSON.parseObject(parameter, OracleDataSource.class); + datasource = JSONUtils.parseObject(parameter, OracleDataSource.class); Class.forName(Constants.COM_ORACLE_JDBC_DRIVER); break; case SQLSERVER: - datasource = JSON.parseObject(parameter, SQLServerDataSource.class); + datasource = JSONUtils.parseObject(parameter, SQLServerDataSource.class); Class.forName(Constants.COM_SQLSERVER_JDBC_DRIVER); break; case DB2: - datasource = JSON.parseObject(parameter, DB2ServerDataSource.class); + datasource = JSONUtils.parseObject(parameter, DB2ServerDataSource.class); Class.forName(Constants.COM_DB2_JDBC_DRIVER); break; default: @@ -526,8 +524,7 @@ public class DataSourceService extends BaseService{ parameterMap.put(Constants.PRINCIPAL,principal); } if (other != null && !"".equals(other)) { - LinkedHashMap map = JSON.parseObject(other, new TypeReference>() { - }); + Map map = JSONUtils.toMap(other); if (map.size() > 0) { StringBuilder otherSb = new StringBuilder(); for (Map.Entry entry: map.entrySet()) { @@ -542,9 +539,9 @@ public class DataSourceService extends BaseService{ } if(logger.isDebugEnabled()){ - logger.info("parameters map-----" + JSON.toJSONString(parameterMap)); + logger.info("parameters map:{}", JSONUtils.toJsonString(parameterMap)); } - return JSON.toJSONString(parameterMap); + return JSONUtils.toJsonString(parameterMap); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 611d3ea7f9..6a8dad4f2a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -24,7 +24,7 @@ import org.apache.dolphinscheduler.common.enums.*; import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; @@ -509,7 +509,7 @@ public class ExecutorService extends BaseService{ if(warningType != null){ command.setWarningType(warningType); } - command.setCommandParam(JSONUtils.toJson(cmdParam)); + command.setCommandParam(JSONUtils.toJsonString(cmdParam)); command.setExecutorId(executorId); command.setWarningGroupId(warningGroupId); command.setProcessInstancePriority(processInstancePriority); @@ -532,7 +532,7 @@ public class ExecutorService extends BaseService{ if(runMode == RunMode.RUN_MODE_SERIAL){ cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, DateUtils.dateToString(start)); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, DateUtils.dateToString(end)); - command.setCommandParam(JSONUtils.toJson(cmdParam)); + command.setCommandParam(JSONUtils.toJsonString(cmdParam)); return processService.createCommand(command); }else if (runMode == RunMode.RUN_MODE_PARALLEL){ List schedules = processService.queryReleaseSchedulerListByProcessDefinitionId(processDefineId); @@ -547,7 +547,7 @@ public class ExecutorService extends BaseService{ for (Date date : listDate) { cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, DateUtils.dateToString(date)); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, DateUtils.dateToString(date)); - command.setCommandParam(JSONUtils.toJson(cmdParam)); + command.setCommandParam(JSONUtils.toJsonString(cmdParam)); processService.createCommand(command); } return listDate.size(); @@ -558,7 +558,7 @@ public class ExecutorService extends BaseService{ runCunt += 1; cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, DateUtils.dateToString(start)); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, DateUtils.dateToString(start)); - command.setCommandParam(JSONUtils.toJson(cmdParam)); + command.setCommandParam(JSONUtils.toJsonString(cmdParam)); processService.createCommand(command); start = DateUtils.getSomeDay(start, 1); } @@ -570,7 +570,7 @@ public class ExecutorService extends BaseService{ processDefineId, schedule); } }else{ - command.setCommandParam(JSONUtils.toJson(cmdParam)); + command.setCommandParam(JSONUtils.toJsonString(cmdParam)); return processService.createCommand(command); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index cbf432d8c0..e23ed0002a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.api.service; -import com.alibaba.fastjson.JSON; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.fasterxml.jackson.core.JsonProcessingException; @@ -636,7 +635,7 @@ public class ProcessDefinitionService extends BaseDAGService { try { out = response.getOutputStream(); buff = new BufferedOutputStream(out); - buff.write(JSON.toJSONString(processDefinitionList).getBytes(StandardCharsets.UTF_8)); + buff.write(JSONUtils.toJsonString(processDefinitionList).getBytes(StandardCharsets.UTF_8)); buff.flush(); buff.close(); } catch (IOException e) { @@ -756,7 +755,7 @@ public class ProcessDefinitionService extends BaseDAGService { public Map importProcessDefinition(User loginUser, MultipartFile file, String currentProjectName) { Map result = new HashMap<>(5); String processMetaJson = FileUtils.file2String(file); - List processMetaList = JSON.parseArray(processMetaJson, ProcessMeta.class); + List processMetaList = JSONUtils.toList(processMetaJson, ProcessMeta.class); //check file content if (CollectionUtils.isEmpty(processMetaList)) { @@ -1340,9 +1339,9 @@ public class ProcessDefinitionService extends BaseDAGService { */ if (taskInstance.getTaskType().equals(TaskType.SUB_PROCESS.name())) { String taskJson = taskInstance.getTaskJson(); - taskNode = JSON.parseObject(taskJson, TaskNode.class); - subProcessId = Integer.parseInt(JSON.parseObject( - taskNode.getParams()).getString(CMDPARAM_SUB_PROCESS_DEFINE_ID)); + taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); + subProcessId = Integer.parseInt(JSONUtils.parseObject( + taskNode.getParams()).path(CMDPARAM_SUB_PROCESS_DEFINE_ID).asText()); } treeViewDto.getInstances().add(new Instance(taskInstance.getId(), taskInstance.getName(), taskInstance.getTaskType(), taskInstance.getState().toString() , taskInstance.getStartTime(), taskInstance.getEndTime(), taskInstance.getHost(), DateUtils.format2Readable(endTime.getTime() - startTime.getTime()), subProcessId)); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java index 7e8a2abeae..8b393a7fe6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java @@ -16,7 +16,8 @@ */ package org.apache.dolphinscheduler.api.service; -import java.nio.charset.StandardCharsets; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.api.dto.gantt.GanttDto; import org.apache.dolphinscheduler.api.dto.gantt.Task; import org.apache.dolphinscheduler.api.enums.Status; @@ -33,11 +34,11 @@ import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; -import com.alibaba.fastjson.JSON; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.dao.entity.*; -import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,6 +50,7 @@ import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.util.*; import java.util.stream.Collectors; @@ -242,7 +244,7 @@ public class ProcessInstanceService extends BaseDAGService { if(logResult.getCode() == Status.SUCCESS.ordinal()){ String log = (String) logResult.getData(); Map resultMap = parseLogForDependentResult(log); - taskInstance.setDependentResult(JSONUtils.toJson(resultMap)); + taskInstance.setDependentResult(JSONUtils.toJsonString(resultMap)); } } } @@ -380,7 +382,7 @@ public class ProcessInstanceService extends BaseDAGService { return result; } - originDefParams = JSONUtils.toJson(processData.getGlobalParams()); + originDefParams = JSONUtils.toJsonString(processData.getGlobalParams()); List globalParamList = processData.getGlobalParams(); Map globalParamMap = globalParamList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); globalParams = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, @@ -529,16 +531,16 @@ public class ProcessInstanceService extends BaseDAGService { List globalParams = new ArrayList<>(); if (userDefinedParams != null && userDefinedParams.length() > 0) { - globalParams = JSON.parseArray(userDefinedParams, Property.class); + globalParams = JSONUtils.toList(userDefinedParams, Property.class); } List taskNodeList = workflowData.getTasks(); // global param string - String globalParamStr = JSONUtils.toJson(globalParams); + String globalParamStr = JSONUtils.toJsonString(globalParams); globalParamStr = ParameterUtils.convertParameterPlaceholders(globalParamStr, timeParams); - globalParams = JSON.parseArray(globalParamStr, Property.class); + globalParams = JSONUtils.toList(globalParamStr, Property.class); for (Property property : globalParams) { timeParams.put(property.getProp(), property.getValue()); } @@ -551,7 +553,8 @@ public class ProcessInstanceService extends BaseDAGService { String localParams = map.get(LOCAL_PARAMS); if (localParams != null && !localParams.isEmpty()) { localParams = ParameterUtils.convertParameterPlaceholders(localParams, timeParams); - List localParamsList = JSON.parseArray(localParams, Property.class); + List localParamsList = JSONUtils.toList(localParams, Property.class); + Map localParamsMap = new HashMap<>(); localParamsMap.put("taskType",taskNode.getType()); localParamsMap.put("localParamsList",localParamsList); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index ef1251964c..7043bc0116 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -16,10 +16,9 @@ */ package org.apache.dolphinscheduler.api.service; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.serializer.SerializerFeature; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.fasterxml.jackson.databind.SerializationFeature; import org.apache.commons.collections.BeanMap; import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; import org.apache.dolphinscheduler.api.dto.resources.filter.ResourceFilter; @@ -544,7 +543,6 @@ public class ResourcesService extends BaseService { } List allResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0); Visitor resourceTreeVisitor = new ResourceTreeVisitor(allResourceList); - //JSONArray jsonArray = JSON.parseArray(JSON.toJSONString(resourceTreeVisitor.visit().getChildren(), SerializerFeature.SortField)); result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren()); putMsg(result,Status.SUCCESS); @@ -1128,8 +1126,9 @@ public class ResourcesService extends BaseService { } List authedResources = resourcesMapper.queryAuthorizedResourceList(userId); Visitor visitor = new ResourceTreeVisitor(authedResources); - logger.info(JSON.toJSONString(visitor.visit(), SerializerFeature.SortField)); - String jsonTreeStr = JSON.toJSONString(visitor.visit().getChildren(), SerializerFeature.SortField); + String visit = JSONUtils.toJsonString(visitor.visit(), SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); + logger.info(visit); + String jsonTreeStr = JSONUtils.toJsonString(visitor.visit().getChildren(), SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); logger.info(jsonTreeStr); result.put(Constants.DATA_LIST, visitor.visit().getChildren()); putMsg(result,Status.SUCCESS); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java index e7af56eaf4..f926fa9d49 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java @@ -24,7 +24,7 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.*; import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.Project; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java index a888712511..9c3bbe9cfd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java @@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.api.utils; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.task.AbstractParameters; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.TaskParametersUtils; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParam.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParam.java index ce43d0ec01..9a4389376b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParam.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParam.java @@ -20,7 +20,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.common.enums.TaskType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AccessTokenControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AccessTokenControllerTest.java index a219343371..57bab355a5 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AccessTokenControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AccessTokenControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AlertGroupControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AlertGroupControllerTest.java index 2656042e51..3c1bb2358c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AlertGroupControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AlertGroupControllerTest.java @@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.AlertType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataAnalysisControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataAnalysisControllerTest.java index 3b9bb117d2..6848daa746 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataAnalysisControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataAnalysisControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Before; import org.junit.Ignore; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java index 5ed7310c47..3d89670883 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecutorControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecutorControllerTest.java index 35df17ebc1..77d8f38df0 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecutorControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ExecutorControllerTest.java @@ -21,7 +21,7 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.FailureStrategy; import org.apache.dolphinscheduler.common.enums.WarningType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoggerControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoggerControllerTest.java index 7718efbc3f..9b118f328c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoggerControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoggerControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java index a4ecd291f9..23e0819268 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java index 41674d3e54..5dd749e5ef 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java index 1e74f0dd99..96fbe4cec1 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java index 7eb0eb3202..0b185c84d3 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/QueueControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/QueueControllerTest.java index d09dd5a7bb..3ed2ba7c35 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/QueueControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/QueueControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java index a56e3f83ef..2843f49f1c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java @@ -16,13 +16,12 @@ */ package org.apache.dolphinscheduler.api.controller; -import com.alibaba.fastjson.JSON; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UdfType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import com.alibaba.fastjson.JSONObject; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -55,7 +54,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -79,7 +78,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -282,7 +281,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -304,7 +303,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -325,7 +324,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -345,7 +344,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -366,7 +365,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -387,7 +386,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -407,7 +406,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -428,7 +427,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -447,7 +446,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java index af7a35dade..8c3090b205 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java @@ -21,7 +21,7 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.FailureStrategy; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.WarningType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java index 674832c118..368981c0d3 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java index ad4a165ca1..4a26bc2607 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java index b3beacfb01..578d6edee1 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java index 0798151299..2e305a0d89 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java index 8517ad2d45..f9a2ffc218 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java @@ -19,7 +19,7 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index d1a051295b..d4fa2307ec 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -25,7 +25,7 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.*; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.FileUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.*; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -521,7 +521,6 @@ public class ProcessDefinitionServiceTest { @Test public void testExportProcessMetaDataStr() { Mockito.when(scheduleMapper.queryByProcessDefinitionId(46)).thenReturn(getSchedulerList()); - ProcessDefinition processDefinition = getProcessDefinition(); processDefinition.setProcessDefinitionJson(sqlDependentJson); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java index ccc231fcf6..50e6323e21 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java @@ -35,7 +35,7 @@ import org.apache.dolphinscheduler.common.task.shell.ShellParameters; import org.apache.dolphinscheduler.common.task.spark.SparkParameters; import org.apache.dolphinscheduler.common.task.sql.SqlParameters; import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.TaskParametersUtils; import org.junit.After; import org.junit.Before; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DataSourceParamTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DataSourceParamTest.java index 4566d93af2..d20b81ff2e 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DataSourceParamTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DataSourceParamTest.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.api.utils.exportprocess; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.api.ApiApplicationServer; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.json.JSONException; import org.junit.Test; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParamTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParamTest.java index be61ab7559..76074d71bf 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParamTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/exportprocess/DependentParamTest.java @@ -16,11 +16,10 @@ */ package org.apache.dolphinscheduler.api.utils.exportprocess; -import com.alibaba.fastjson.JSONObject; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.api.ApiApplicationServer; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.json.JSONException; import org.junit.Test; diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index 4d4ccede26..28acb21fd5 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -36,11 +36,7 @@ org.apache.dolphinscheduler dolphinscheduler-plugin-api - - com.alibaba - fastjson - compile - + org.apache.httpcomponents httpclient diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/DateInterval.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/DateInterval.java index e936a444d0..2ff3922c18 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/DateInterval.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/DateInterval.java @@ -16,6 +16,8 @@ */ package org.apache.dolphinscheduler.common.model; +import com.fasterxml.jackson.annotation.JsonFormat; + import java.util.Date; import java.util.Objects; @@ -23,9 +25,9 @@ import java.util.Objects; * date interval class */ public class DateInterval { - + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; - + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date endTime; public DateInterval(Date beginTime, Date endTime){ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/Server.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/Server.java index 706609df56..69d718c286 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/Server.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/Server.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.common.model; +import com.fasterxml.jackson.annotation.JsonFormat; + import java.util.Date; /** @@ -52,11 +54,13 @@ public class Server { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * laster heart beat time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date lastHeartbeatTime; public int getId() { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java index 35767a0a46..5ab970e69b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java @@ -16,14 +16,13 @@ */ package org.apache.dolphinscheduler.common.model; -import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -195,7 +194,7 @@ public class TaskNode { public void setDepList(List depList) throws JsonProcessingException { this.depList = depList; - this.preTasks = JSONUtils.toJson(depList); + this.preTasks = JSONUtils.toJsonString(depList); } public String getLoc() { @@ -295,7 +294,7 @@ public class TaskNode { if(StringUtils.isNotEmpty(this.getTimeout())){ String formatStr = String.format("%s,%s", TaskTimeoutStrategy.WARN.name(), TaskTimeoutStrategy.FAILED.name()); String taskTimeout = this.getTimeout().replace(formatStr,TaskTimeoutStrategy.WARNFAILED.name()); - return JSON.parseObject(taskTimeout,TaskTimeoutParameter.class); + return JSONUtils.parseObject(taskTimeout,TaskTimeoutParameter.class); } return new TaskTimeoutParameter(false); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 1544b449ff..71bbf1ad94 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -16,9 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONException; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; @@ -28,8 +26,8 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.*; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.client.cli.RMAdminCLI; import org.slf4j.Logger; @@ -408,9 +406,8 @@ public class HadoopUtils implements Closeable { * * @param applicationId application id * @return the return may be null or there may be other parse exceptions - * @throws JSONException json exception */ - public ExecutionStatus getApplicationStatus(String applicationId) throws JSONException { + public ExecutionStatus getApplicationStatus(String applicationId) { if (StringUtils.isEmpty(applicationId)) { return null; } @@ -421,15 +418,18 @@ public class HadoopUtils implements Closeable { String responseContent = HttpUtils.get(applicationUrl); if (responseContent != null) { - JSONObject jsonObject = JSON.parseObject(responseContent); - result = jsonObject.getJSONObject("app").getString("finalStatus"); + ObjectNode jsonObject = JSONUtils.parseObject(responseContent); + result = jsonObject.path("app").path("finalStatus").asText(); } else { //may be in job history String jobHistoryUrl = getJobHistoryUrl(applicationId); logger.info("jobHistoryUrl={}", jobHistoryUrl); responseContent = HttpUtils.get(jobHistoryUrl); - JSONObject jsonObject = JSONObject.parseObject(responseContent); - result = jsonObject.getJSONObject("job").getString("state"); + ObjectNode jsonObject = JSONUtils.parseObject(responseContent); + if (!jsonObject.has("job")){ + return ExecutionStatus.FAILURE; + } + result = jsonObject.path("job").path("state").asText(); } switch (result) { @@ -469,6 +469,7 @@ public class HadoopUtils implements Closeable { * hdfs resource dir * * @param tenantCode tenant code + * @param resourceType resource type * @return hdfs resource dir */ public static String getHdfsDir(ResourceType resourceType, String tenantCode) { @@ -669,10 +670,13 @@ public class HadoopUtils implements Closeable { return null; } //to json - JSONObject jsonObject = JSON.parseObject(retStr); + ObjectNode jsonObject = JSONUtils.parseObject(retStr); //get ResourceManager state - return jsonObject.getJSONObject("clusterInfo").getString("haState"); + if (!jsonObject.has("clusterInfo")){ + return null; + } + return jsonObject.get("clusterInfo").path("haState").asText(); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index 4f701490e5..20209c0a98 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.*; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.type.CollectionType; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,15 +51,30 @@ public class JSONUtils { objectMapper.configure(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true).setTimeZone(TimeZone.getDefault()); } + + public static ArrayNode createArrayNode() { + return objectMapper.createArrayNode(); + } + + public static ObjectNode createObjectNode() { + return objectMapper.createObjectNode(); + } + + public static JsonNode toJsonNode(Object obj) { + return objectMapper.valueToTree(obj); + } + /** * json representation of object * * @param object object + * @param feature feature * @return object to json string */ - public static String toJson(Object object) { + public static String toJsonString(Object object, SerializationFeature feature) { try { - return objectMapper.writeValueAsString(object); + ObjectWriter writer = objectMapper.writer(feature); + return writer.writeValueAsString(object); } catch (Exception e) { logger.error("object to json exception!", e); } @@ -66,7 +82,6 @@ public class JSONUtils { return null; } - /** * This method deserializes the specified Json into an object of the specified class. It is not * suitable to use if the specified class is a generic type since it will not have the generic @@ -94,7 +109,6 @@ public class JSONUtils { return null; } - /** * json to list * @@ -105,16 +119,18 @@ public class JSONUtils { */ public static List toList(String json, Class clazz) { if (StringUtils.isEmpty(json)) { - return new ArrayList<>(); + return Collections.emptyList(); } + try { - return objectMapper.readValue(json, new TypeReference>() { - }); + + CollectionType listType = objectMapper.getTypeFactory().constructCollectionType(ArrayList.class, clazz); + return objectMapper.readValue(json, listType); } catch (Exception e) { - logger.error("JSONArray.parseArray exception!", e); + logger.error("parse list exception!", e); } - return new ArrayList<>(); + return Collections.emptyList(); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java index f3efef2eca..aae14a63ec 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java @@ -16,7 +16,8 @@ */ package org.apache.dolphinscheduler.common.utils; -import com.alibaba.fastjson.JSON; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.time.DateUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DataType; @@ -24,8 +25,6 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils; import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.time.DateUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -196,7 +195,7 @@ public class ParameterUtils { property.setValue(val); } } - return JSONUtils.toJson(globalParamList); + return JSONUtils.toJsonString(globalParamList); } @@ -216,9 +215,10 @@ public class ParameterUtils { /** * new * $[yyyyMMdd] replace scheduler time - * @param text - * @param paramsMap - * @return + * @param text text + * @param scheduleTime scheduleTime + * @param paramsMap paramsMap + * @return text */ public static String replaceScheduleTime(String text, Date scheduleTime, Map paramsMap) { if (paramsMap != null) { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java index feadb68ee6..c4ce006b0c 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java @@ -85,7 +85,7 @@ public class ResInfo { */ public static String getResInfoJson(double cpuUsage , double memoryUsage,double loadAverage){ ResInfo resInfo = new ResInfo(cpuUsage,memoryUsage,loadAverage); - return JSONUtils.toJson(resInfo); + return JSONUtils.toJsonString(resInfo); } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/HttpParametersTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/HttpParametersTest.java index abcfca83dd..6bd3bc76b5 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/HttpParametersTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/HttpParametersTest.java @@ -16,20 +16,13 @@ */ package org.apache.dolphinscheduler.common.task; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.enums.HttpCheckCondition; import org.apache.dolphinscheduler.common.enums.HttpMethod; -import org.apache.dolphinscheduler.common.process.HttpProperty; -import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.http.HttpParameters; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.junit.Assert; import org.junit.Test; -import com.alibaba.fastjson.JSON; - /** * http parameter */ @@ -40,7 +33,7 @@ public class HttpParametersTest { public void testGenerator(){ String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," + "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}"; - HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class); + HttpParameters httpParameters = JSONUtils.parseObject(paramData, HttpParameters.class); Assert.assertEquals(10000,httpParameters.getConnectTimeout() ); @@ -57,7 +50,7 @@ public class HttpParametersTest { public void testCheckParameters(){ String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," + "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}"; - HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class); + HttpParameters httpParameters = JSONUtils.parseObject(paramData, HttpParameters.class); Assert.assertTrue( httpParameters.checkParameters()); Assert.assertEquals(10000,httpParameters.getConnectTimeout() ); @@ -74,7 +67,7 @@ public class HttpParametersTest { public void testCheckValues() { String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," + "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}"; - HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class); + HttpParameters httpParameters = JSONUtils.parseObject(paramData, HttpParameters.class); Assert.assertTrue( httpParameters.checkParameters()); Assert.assertEquals(10000,httpParameters.getConnectTimeout() ); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java index 17929f9344..90ed11aed5 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java @@ -16,8 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -37,8 +36,8 @@ public class HttpUtilsTest { //success String result = HttpUtils.get("https://github.com/manifest.json"); Assert.assertNotNull(result); - JSONObject jsonObject = JSON.parseObject(result); - Assert.assertEquals("GitHub", jsonObject.getString("name")); + ObjectNode jsonObject = JSONUtils.parseObject(result); + Assert.assertEquals("GitHub", jsonObject.path("name").asText()); result = HttpUtils.get("https://123.333.111.33/ccc"); Assert.assertNull(result); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java index 1756078fe1..eca392d9e2 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java @@ -16,53 +16,95 @@ */ package org.apache.dolphinscheduler.common.utils; -import com.alibaba.fastjson.JSON; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.Direct; import org.apache.dolphinscheduler.common.process.Property; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; import org.junit.Assert; import org.junit.Test; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import java.util.*; public class JSONUtilsTest { + @Test + public void createArrayNodeTest() { + Property property = new Property(); + property.setProp("ds"); + property.setDirect(Direct.IN); + property.setType(DataType.VARCHAR); + property.setValue("sssssss"); + String str = "[{\"prop\":\"ds\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"},{\"prop\":\"ds\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"}]"; + JsonNode jsonNode = JSONUtils.toJsonNode(property); + + ArrayNode arrayNode = JSONUtils.createArrayNode(); + ArrayList objects = new ArrayList<>(); + objects.add(jsonNode); + objects.add(jsonNode); + + ArrayNode jsonNodes = arrayNode.addAll(objects); + String s = JSONUtils.toJsonString(jsonNodes); + Assert.assertEquals(s, str); + + } + + @Test + public void toJsonNodeTest() { + Property property = new Property(); + property.setProp("ds"); + property.setDirect(Direct.IN); + property.setType(DataType.VARCHAR); + property.setValue("sssssss"); + String str = "{\"prop\":\"ds\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"}"; + + JsonNode jsonNodes = JSONUtils.toJsonNode(property); + String s = JSONUtils.toJsonString(jsonNodes); + Assert.assertEquals(s, str); + + } + + @Test + public void createObjectNodeTest() { + String jsonStr = "{\"a\":\"b\",\"b\":\"d\"}"; + + ObjectNode objectNode = JSONUtils.createObjectNode(); + objectNode.put("a","b"); + objectNode.put("b","d"); + String s = JSONUtils.toJsonString(objectNode); + Assert.assertEquals(s, jsonStr); + } @Test public void toMap() { String jsonStr = "{\"id\":\"1001\",\"name\":\"Jobs\"}"; - Map models = JSONUtils.toMap(jsonStr); + Map models = JSONUtils.toMap(jsonStr); Assert.assertEquals("1001", models.get("id")); Assert.assertEquals("Jobs", models.get("name")); } @Test - public void convert2Property(){ + public void convert2Property() { Property property = new Property(); property.setProp("ds"); property.setDirect(Direct.IN); property.setType(DataType.VARCHAR); property.setValue("sssssss"); String str = "{\"direct\":\"IN\",\"prop\":\"ds\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"}"; - Property property1 = JSON.parseObject(str, Property.class); + Property property1 = JSONUtils.parseObject(str, Property.class); Direct direct = property1.getDirect(); Assert.assertEquals(Direct.IN, direct); } @Test - public void String2MapTest(){ + public void String2MapTest() { String str = list2String(); List maps = JSONUtils.toList(str, @@ -76,34 +118,24 @@ public class JSONUtilsTest { Assert.assertEquals("190", maps.get(0).get("database client connections")); } - public String list2String(){ + public String list2String() { LinkedHashMap map1 = new LinkedHashMap<>(); - map1.put("mysql service name","mysql200"); - map1.put("mysql address","192.168.xx.xx"); - map1.put("port","3306"); - map1.put("no index of number","80"); - map1.put("database client connections","190"); + map1.put("mysql service name", "mysql200"); + map1.put("mysql address", "192.168.xx.xx"); + map1.put("port", "3306"); + map1.put("no index of number", "80"); + map1.put("database client connections", "190"); List> maps = new ArrayList<>(); - maps.add(0,map1); - String resultJson = JSONUtils.toJson(maps); + maps.add(0, map1); + String resultJson = JSONUtils.toJsonString(maps); return resultJson; } - @Test - public void testToJson() { - Map map = new HashMap<>(); - map.put("foo","bar"); - - Assert.assertEquals("{\"foo\":\"bar\"}", JSONUtils.toJson(map)); - Assert.assertEquals( - String.valueOf((Object) null), JSONUtils.toJson(null)); - } - @Test public void testParseObject() { - Assert.assertNull(JSONUtils.parseObject("", null)); + Assert.assertNull(JSONUtils.parseObject("")); Assert.assertNull(JSONUtils.parseObject("foo", String.class)); } @@ -130,7 +162,7 @@ public class JSONUtilsTest { @Test public void testToMap() { Map map = new HashMap<>(); - map.put("foo","bar"); + map.put("foo", "bar"); Assert.assertTrue(map.equals(JSONUtils.toMap( "{\n" + "\"foo\": \"bar\"\n" + "}"))); @@ -157,6 +189,9 @@ public class JSONUtilsTest { JSONUtils.toJsonString(map)); Assert.assertEquals(String.valueOf((Object) null), JSONUtils.toJsonString(null)); + + Assert.assertEquals("{\"foo\":\"bar\"}", + JSONUtils.toJsonString(map, SerializationFeature.WRITE_NULL_MAP_VALUES)); } @Test diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java index 3d40e7a4cf..8627226b8b 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.common.utils; -import com.alibaba.fastjson.JSON; import org.apache.commons.lang.time.DateUtils; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DataType; @@ -27,7 +26,9 @@ import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.util.*; + import static org.apache.dolphinscheduler.common.Constants.PARAMETER_FORMAT_TIME; import static org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils.replacePlaceholders; @@ -91,13 +92,13 @@ public class ParameterUtilsTest { globalParamList.add(property); String result2 = ParameterUtils.curingGlobalParams(null,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,scheduleTime); - Assert.assertEquals(result2, JSONUtils.toJson(globalParamList)); + Assert.assertEquals(result2, JSONUtils.toJsonString(globalParamList)); String result3 = ParameterUtils.curingGlobalParams(globalParamMap,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,null); - Assert.assertEquals(result3, JSONUtils.toJson(globalParamList)); + Assert.assertEquals(result3, JSONUtils.toJsonString(globalParamList)); String result4 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime); - Assert.assertEquals(result4, JSONUtils.toJson(globalParamList)); + Assert.assertEquals(result4, JSONUtils.toJsonString(globalParamList)); //test var $ startsWith globalParamMap.put("bizDate","${system.biz.date}"); @@ -113,7 +114,7 @@ public class ParameterUtilsTest { globalParamList.add(property4); String result5 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime); - Assert.assertEquals(result5, JSONUtils.toJson(globalParamList)); + Assert.assertEquals(result5, JSONUtils.toJsonString(globalParamList)); } /** diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index 3aea888f94..b7fc8725e9 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -122,12 +122,6 @@ commons-httpclient - - com.alibaba - fastjson - compile - - com.cronutils cron-utils diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java index cca1fa041d..53f2468ea6 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CycleDependency.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CycleDependency.java index e7ad38b6ea..89a57aae95 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CycleDependency.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CycleDependency.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.dao.entity; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.CycleEnum; import java.util.Date; @@ -31,10 +32,12 @@ public class CycleDependency { /** * last schedule time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date lastScheduleTime; /** * expiration time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date expirationTime; /** * cycle enum diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java index 236ae8f91e..ecd5be0455 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.dao.entity; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.DbType; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; @@ -66,11 +67,13 @@ public class DataSource { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; public DataSource() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java index 127c5b7322..760bb23d90 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.dao.entity; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.*; import java.util.Date; @@ -78,11 +79,13 @@ public class ErrorCommand { /** * schedule time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date scheduleTime; /** * start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** @@ -93,6 +96,7 @@ public class ErrorCommand { /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/MonitorRecord.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/MonitorRecord.java index 259827ed80..318f24535f 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/MonitorRecord.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/MonitorRecord.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.dao.entity; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.Flag; @@ -56,6 +57,7 @@ public class MonitorRecord { /** * start date */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date date; public Flag getState() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java index e29de897ef..3648482996 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java @@ -16,16 +16,18 @@ */ package org.apache.dolphinscheduler.dao.entity; -import com.alibaba.fastjson.JSON; -import org.apache.dolphinscheduler.common.enums.Flag; -import org.apache.dolphinscheduler.common.enums.ReleaseState; -import org.apache.dolphinscheduler.common.process.Property; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.core.toolkit.StringUtils; +import com.fasterxml.jackson.annotation.JsonFormat; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.ReleaseState; +import org.apache.dolphinscheduler.common.process.Property; +import org.apache.dolphinscheduler.common.utils.*; +import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; @@ -37,10 +39,11 @@ import java.util.stream.Collectors; */ @TableName("t_ds_process_definition") public class ProcessDefinition { + /** * id */ - @TableId(value="id", type=IdType.AUTO) + @TableId(value = "id", type = IdType.AUTO) private int id; /** @@ -81,23 +84,25 @@ public class ProcessDefinition { /** * user defined parameter list */ - @TableField(exist=false) + @TableField(exist = false) private List globalParamList; /** * user define parameter map */ - @TableField(exist=false) - private Map globalParamMap; + @TableField(exist = false) + private Map globalParamMap; /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; /** @@ -145,7 +150,7 @@ public class ProcessDefinition { /** * schedule release state : online/offline */ - @TableField(exist=false) + @TableField(exist = false) private ReleaseState scheduleReleaseState; /** @@ -271,7 +276,11 @@ public class ProcessDefinition { } public void setGlobalParams(String globalParams) { - this.globalParamList = JSON.parseArray(globalParams, Property.class); + if (globalParams == null){ + this.globalParamList = new ArrayList<>(); + }else { + this.globalParamList = JSONUtils.toList(globalParams, Property.class); + } this.globalParams = globalParams; } @@ -280,15 +289,13 @@ public class ProcessDefinition { } public void setGlobalParamList(List globalParamList) { - this.globalParams = JSON.toJSONString(globalParamList); + this.globalParams = JSONUtils.toJsonString(globalParamList); this.globalParamList = globalParamList; } public Map getGlobalParamMap() { - List propList; - if (globalParamMap == null && StringUtils.isNotEmpty(globalParams)) { - propList = JSON.parseArray(globalParams, Property.class); + List propList = JSONUtils.toList(globalParams,Property.class); globalParamMap = propList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java index 2fa8e64451..28f1eba975 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java @@ -21,6 +21,7 @@ import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.core.toolkit.StringUtils; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.*; import java.util.Date; @@ -52,11 +53,13 @@ public class ProcessInstance { /** * start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** * end time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date endTime; /** @@ -117,11 +120,13 @@ public class ProcessInstance { /** * schedule time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date scheduleTime; /** * command start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date commandStartTime; /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Queue.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Queue.java index 57b678d2a7..9c89167d76 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Queue.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Queue.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.dao.entity; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; import java.util.Date; @@ -45,10 +46,12 @@ public class Queue { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; public int getId() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java index 16d94914fd..214fbe83e4 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.dao.entity; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.ResourceType; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; @@ -80,11 +81,13 @@ public class Resource { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; public Resource() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesUser.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesUser.java index dc62a7b66f..ea8bd6c1b9 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesUser.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesUser.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.dao.entity; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; import java.util.Date; @@ -52,11 +53,13 @@ public class ResourcesUser { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; public int getId() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java index 0cb41080b2..5d2bc3811e 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java @@ -20,6 +20,7 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.FailureStrategy; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.ReleaseState; @@ -62,11 +63,13 @@ public class Schedule { /** * schedule start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** * schedule end time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date endTime; /** @@ -87,11 +90,13 @@ public class Schedule { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Session.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Session.java index 39bf6143dd..060d8824d3 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Session.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Session.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.dao.entity; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; import java.util.Date; @@ -42,6 +43,7 @@ public class Session { /** * last login time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date lastLoginTime; /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java index 53b56e54b2..eb51fc50cb 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java @@ -17,12 +17,13 @@ package org.apache.dolphinscheduler.dao.entity; import com.baomidou.mybatisplus.annotation.TableField; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; @@ -84,16 +85,19 @@ public class TaskInstance implements Serializable { /** * task submit time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date submitTime; /** * task start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** * task end time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date endTime; /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskRecord.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskRecord.java index 9a6979525b..b267dff80c 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskRecord.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskRecord.java @@ -16,6 +16,8 @@ */ package org.apache.dolphinscheduler.dao.entity; +import com.fasterxml.jackson.annotation.JsonFormat; + import java.util.Date; /** @@ -46,11 +48,13 @@ public class TaskRecord { /** * start date */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** * end date */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date endTime; /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Tenant.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Tenant.java index a289f4f1f5..1b2eea403d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Tenant.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Tenant.java @@ -20,6 +20,7 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; import java.util.Date; import java.util.Objects; @@ -71,10 +72,12 @@ public class Tenant { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UDFUser.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UDFUser.java index c21eb96abd..fa3fbc2771 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UDFUser.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UDFUser.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.dao.entity; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; import java.util.Date; @@ -52,11 +53,13 @@ public class UDFUser { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; public int getId() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java index e14255be77..ef0badf7e5 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.dao.entity; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.UdfType; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; @@ -81,11 +82,13 @@ public class UdfFunc { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; public int getId() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java index b4f2f93dcc..89da171caf 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.dao.entity; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.UserType; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; @@ -104,11 +105,13 @@ public class User { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; public int getId() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UserAlertGroup.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UserAlertGroup.java index e86b32e075..902196d186 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UserAlertGroup.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UserAlertGroup.java @@ -20,6 +20,7 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; import java.util.Date; @@ -61,11 +62,13 @@ public class UserAlertGroup { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * update time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; public int getId() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerGroup.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerGroup.java index bce963686c..b850219836 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerGroup.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerGroup.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.dao.entity; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; import java.util.Date; import java.util.List; @@ -32,8 +33,9 @@ public class WorkerGroup { private List ipList; + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; - + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date updateTime; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerServer.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerServer.java index 977302061b..58bf2d24d5 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerServer.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerServer.java @@ -16,6 +16,8 @@ */ package org.apache.dolphinscheduler.dao.entity; +import com.fasterxml.jackson.annotation.JsonFormat; + import java.util.Date; public class WorkerServer { @@ -49,11 +51,13 @@ public class WorkerServer { /** * create time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date createTime; /** * last heart beat time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date lastHeartbeatTime; public int getId() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java index 1133cadbe7..4418cce7d8 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java @@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.ProcessDag; import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.ProcessData; import org.apache.dolphinscheduler.dao.entity.TaskInstance; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java new file mode 100644 index 0000000000..951311672a --- /dev/null +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.dao.entity; + +import org.junit.Assert; +import org.junit.Test; + +public class ProcessDefinitionTest { + + /** + * task instance sub process + */ + @Test + public void getGlobalParamMapTest() { + ProcessDefinition taskInstance = new ProcessDefinition(); + + //sub process + taskInstance.setGlobalParams("[{\"prop\":\"selenium_global_parameters_1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"selenium_global_parameters_value_1\"}]"); + + taskInstance.getGlobalParamMap(); + Assert.assertEquals("{selenium_global_parameters_1=selenium_global_parameters_value_1}",taskInstance.getGlobalParamMap().toString()); + + + + } +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java index 9dafbe138c..ec995df51c 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java @@ -69,6 +69,8 @@ public class ProcessDefinitionMapperTest { processDefinition.setUserId(101); processDefinition.setUpdateTime(new Date()); processDefinition.setCreateTime(new Date()); +// processDefinition.setGlobalParams("[{\"prop\":\"selenium_global_parameters_1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"selenium_global_parameters_value_1\"}]"); + processDefinitionMapper.insert(processDefinition); return processDefinition; } @@ -143,6 +145,8 @@ public class ProcessDefinitionMapperTest { processDefinition.setCreateTime(new Date()); processDefinition.setTenantId(tenant.getId()); processDefinition.setUserId(user.getId()); +// processDefinition.setGlobalParams("[{\"prop\":\"selenium_global_parameters_1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"selenium_global_parameters_value_1\"}]"); +// processDefinitionMapper.insert(processDefinition); ProcessDefinition processDefinition1 = processDefinitionMapper.queryByDefineName(project.getId(), "def 1"); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java index 95c7d2f086..14dfe0b750 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java @@ -24,7 +24,7 @@ import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.ProcessDag; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.ProcessData; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.junit.Assert; diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 52563f5573..f90a346d37 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -255,7 +255,6 @@ The text of each license is also included at licenses/LICENSE-[project].txt. derby 10.14.2.0: https://github.com/apache/derby, Apache 2.0 druid 1.1.14: https://mvnrepository.com/artifact/com.alibaba/druid/1.1.14, Apache 2.0 error_prone_annotations 2.1.2: https://mvnrepository.com/artifact/com.google.errorprone/error_prone_annotations/2.1.2, Apache 2.0 - fastjson 1.2.61: https://mvnrepository.com/artifact/com.alibaba/fastjson/1.2.61, Apache 2.0 freemarker 2.3.21: https://github.com/apache/freemarker, Apache 2.0 grpc-context 1.9.0: https://mvnrepository.com/artifact/io.grpc/grpc-context/1.9.0, Apache 2.0 grpc-core 1.9.0: https://mvnrepository.com/artifact/io.grpc/grpc-core/1.9.0, Apache 2.0 diff --git a/dolphinscheduler-dist/release-docs/NOTICE b/dolphinscheduler-dist/release-docs/NOTICE index ecab936a4f..ca2f914701 100644 --- a/dolphinscheduler-dist/release-docs/NOTICE +++ b/dolphinscheduler-dist/release-docs/NOTICE @@ -2214,26 +2214,6 @@ The Apache Software Foundation (http://www.apache.org/). ========================================================================= -FastJson NOTICE -========================================================================= -/* - * Copyright 1999-2017 Alibaba Group. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -========================================================================= - - Datanucleus NOTICE ========================================================================= diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml index 0968e610bc..c01434c51f 100644 --- a/dolphinscheduler-remote/pom.xml +++ b/dolphinscheduler-remote/pom.xml @@ -43,15 +43,15 @@ org.slf4j slf4j-api - - com.alibaba - fastjson - junit junit test + + com.fasterxml.jackson.core + jackson-databind + diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteAckCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteAckCommand.java index 0b3d901a3c..135c14975b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteAckCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteAckCommand.java @@ -1 +1,133 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.Date; /** * execute task request command */ public class TaskExecuteAckCommand implements Serializable { /** * taskInstanceId */ private int taskInstanceId; /** * startTime */ private Date startTime; /** * host */ private String host; /** * status */ private int status; /** * logPath */ private String logPath; /** * executePath */ private String executePath; public Date getStartTime() { return startTime; } public void setStartTime(Date startTime) { this.startTime = startTime; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } public String getLogPath() { return logPath; } public void setLogPath(String logPath) { this.logPath = logPath; } public String getExecutePath() { return executePath; } public void setExecutePath(String executePath) { this.executePath = executePath; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_EXECUTE_ACK); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskExecuteAckCommand{" + "taskInstanceId=" + taskInstanceId + ", startTime=" + startTime + ", host='" + host + '\'' + ", status=" + status + ", logPath='" + logPath + '\'' + ", executePath='" + executePath + '\'' + '}'; } } \ No newline at end of file +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.command; + +import com.fasterxml.jackson.annotation.JsonFormat; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; + +import java.io.Serializable; +import java.util.Date; + +/** + * execute task request command + */ +public class TaskExecuteAckCommand implements Serializable { + + /** + * taskInstanceId + */ + private int taskInstanceId; + + /** + * startTime + */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + private Date startTime; + + /** + * host + */ + private String host; + + /** + * status + */ + private int status; + + /** + * logPath + */ + private String logPath; + + /** + * executePath + */ + private String executePath; + + public Date getStartTime() { + return startTime; + } + + public void setStartTime(Date startTime) { + this.startTime = startTime; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + public int getTaskInstanceId() { + return taskInstanceId; + } + + public void setTaskInstanceId(int taskInstanceId) { + this.taskInstanceId = taskInstanceId; + } + + public String getLogPath() { + return logPath; + } + + public void setLogPath(String logPath) { + this.logPath = logPath; + } + + public String getExecutePath() { + return executePath; + } + + public void setExecutePath(String executePath) { + this.executePath = executePath; + } + + /** + * package request command + * + * @return command + */ + public Command convert2Command(){ + Command command = new Command(); + command.setType(CommandType.TASK_EXECUTE_ACK); + byte[] body = JsonSerializer.serialize(this); + command.setBody(body); + return command; + } + + @Override + public String toString() { + return "TaskExecuteAckCommand{" + + "taskInstanceId=" + taskInstanceId + + ", startTime=" + startTime + + ", host='" + host + '\'' + + ", status=" + status + + ", logPath='" + logPath + '\'' + + ", executePath='" + executePath + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java index 637724f49d..4ae28e3ca5 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java @@ -1 +1,67 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; /** * execute task request command */ public class TaskExecuteRequestCommand implements Serializable { /** * task execution context */ private String taskExecutionContext; public String getTaskExecutionContext() { return taskExecutionContext; } public void setTaskExecutionContext(String taskExecutionContext) { this.taskExecutionContext = taskExecutionContext; } public TaskExecuteRequestCommand() { } public TaskExecuteRequestCommand(String taskExecutionContext) { this.taskExecutionContext = taskExecutionContext; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_EXECUTE_REQUEST); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskExecuteRequestCommand{" + "taskExecutionContext='" + taskExecutionContext + '\'' + '}'; } } \ No newline at end of file +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.command; + +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; + +import java.io.Serializable; + +/** + * execute task request command + */ +public class TaskExecuteRequestCommand implements Serializable { + + /** + * task execution context + */ + private String taskExecutionContext; + + public String getTaskExecutionContext() { + return taskExecutionContext; + } + + public void setTaskExecutionContext(String taskExecutionContext) { + this.taskExecutionContext = taskExecutionContext; + } + + public TaskExecuteRequestCommand() { + } + + public TaskExecuteRequestCommand(String taskExecutionContext) { + this.taskExecutionContext = taskExecutionContext; + } + + /** + * package request command + * + * @return command + */ + public Command convert2Command(){ + Command command = new Command(); + command.setType(CommandType.TASK_EXECUTE_REQUEST); + byte[] body = JsonSerializer.serialize(this); + command.setBody(body); + return command; + } + + @Override + public String toString() { + return "TaskExecuteRequestCommand{" + + "taskExecutionContext='" + taskExecutionContext + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java index deb6f5dd8f..e559334f48 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java @@ -1 +1,129 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.Date; /** * execute task response command */ public class TaskExecuteResponseCommand implements Serializable { public TaskExecuteResponseCommand() { } public TaskExecuteResponseCommand(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } /** * task instance id */ private int taskInstanceId; /** * status */ private int status; /** * end time */ private Date endTime; /** * processId */ private int processId; /** * appIds */ private String appIds; public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public Date getEndTime() { return endTime; } public void setEndTime(Date endTime) { this.endTime = endTime; } public int getProcessId() { return processId; } public void setProcessId(int processId) { this.processId = processId; } public String getAppIds() { return appIds; } public void setAppIds(String appIds) { this.appIds = appIds; } /** * package response command * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_EXECUTE_RESPONSE); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskExecuteResponseCommand{" + "taskInstanceId=" + taskInstanceId + ", status=" + status + ", endTime=" + endTime + ", processId=" + processId + ", appIds='" + appIds + '\'' + '}'; } } \ No newline at end of file +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.command; + +import com.fasterxml.jackson.annotation.JsonFormat; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; + +import java.io.Serializable; +import java.util.Date; + +/** + * execute task response command + */ +public class TaskExecuteResponseCommand implements Serializable { + + + public TaskExecuteResponseCommand() { + } + + public TaskExecuteResponseCommand(int taskInstanceId) { + this.taskInstanceId = taskInstanceId; + } + + /** + * task instance id + */ + private int taskInstanceId; + + /** + * status + */ + private int status; + + + /** + * end time + */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + private Date endTime; + + + /** + * processId + */ + private int processId; + + /** + * appIds + */ + private String appIds; + + + public int getTaskInstanceId() { + return taskInstanceId; + } + + public void setTaskInstanceId(int taskInstanceId) { + this.taskInstanceId = taskInstanceId; + } + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + public Date getEndTime() { + return endTime; + } + + public void setEndTime(Date endTime) { + this.endTime = endTime; + } + + public int getProcessId() { + return processId; + } + + public void setProcessId(int processId) { + this.processId = processId; + } + + public String getAppIds() { + return appIds; + } + + public void setAppIds(String appIds) { + this.appIds = appIds; + } + + /** + * package response command + * @return command + */ + public Command convert2Command(){ + Command command = new Command(); + command.setType(CommandType.TASK_EXECUTE_RESPONSE); + byte[] body = JsonSerializer.serialize(this); + command.setBody(body); + return command; + } + + @Override + public String toString() { + return "TaskExecuteResponseCommand{" + + "taskInstanceId=" + taskInstanceId + + ", status=" + status + + ", endTime=" + endTime + + ", processId=" + processId + + ", appIds='" + appIds + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskInfo.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskInfo.java index 3fb58fe3da..196d0a73ae 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskInfo.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskInfo.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.remote.command; +import com.fasterxml.jackson.annotation.JsonFormat; + import java.io.Serializable; import java.util.Date; @@ -39,6 +41,7 @@ public class TaskInfo implements Serializable{ /** * task start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** @@ -66,6 +69,7 @@ public class TaskInfo implements Serializable{ /** * process instance schedule time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date scheduleTime; /** diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java index b8e02dd057..4c0830b7cf 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java @@ -1 +1,61 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; /** * kill task request command */ public class TaskKillRequestCommand implements Serializable { /** * task id */ private int taskInstanceId; public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_KILL_REQUEST); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskKillRequestCommand{" + "taskInstanceId=" + taskInstanceId + '}'; } } \ No newline at end of file +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.command; + +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; + +import java.io.Serializable; + +/** + * kill task request command + */ +public class TaskKillRequestCommand implements Serializable { + + /** + * task id + */ + private int taskInstanceId; + + + public int getTaskInstanceId() { + return taskInstanceId; + } + + public void setTaskInstanceId(int taskInstanceId) { + this.taskInstanceId = taskInstanceId; + } + + /** + * package request command + * + * @return command + */ + public Command convert2Command(){ + Command command = new Command(); + command.setType(CommandType.TASK_KILL_REQUEST); + byte[] body = JsonSerializer.serialize(this); + command.setBody(body); + return command; + } + + @Override + public String toString() { + return "TaskKillRequestCommand{" + + "taskInstanceId=" + taskInstanceId + + '}'; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java index 2ca2330c80..4b48c1ef4f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java @@ -1 +1,119 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.Date; import java.util.List; /** * kill task response command */ public class TaskKillResponseCommand implements Serializable { /** * taskInstanceId */ private int taskInstanceId; /** * host */ private String host; /** * status */ private int status; /** * processId */ private int processId; /** * other resource manager appId , for example : YARN etc */ protected List appIds; public int getTaskInstanceId() { return taskInstanceId; } public void setTaskInstanceId(int taskInstanceId) { this.taskInstanceId = taskInstanceId; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public int getProcessId() { return processId; } public void setProcessId(int processId) { this.processId = processId; } public List getAppIds() { return appIds; } public void setAppIds(List appIds) { this.appIds = appIds; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.TASK_KILL_RESPONSE); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } @Override public String toString() { return "TaskKillResponseCommand{" + "taskInstanceId=" + taskInstanceId + ", host='" + host + '\'' + ", status=" + status + ", processId=" + processId + ", appIds=" + appIds + '}'; } } \ No newline at end of file +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.command; + +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; + +import java.io.Serializable; +import java.util.List; + +/** + * kill task response command + */ +public class TaskKillResponseCommand implements Serializable { + + /** + * taskInstanceId + */ + private int taskInstanceId; + + /** + * host + */ + private String host; + + /** + * status + */ + private int status; + + + /** + * processId + */ + private int processId; + + /** + * other resource manager appId , for example : YARN etc + */ + protected List appIds; + + + public int getTaskInstanceId() { + return taskInstanceId; + } + + public void setTaskInstanceId(int taskInstanceId) { + this.taskInstanceId = taskInstanceId; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + public int getProcessId() { + return processId; + } + + public void setProcessId(int processId) { + this.processId = processId; + } + + public List getAppIds() { + return appIds; + } + + public void setAppIds(List appIds) { + this.appIds = appIds; + } + + /** + * package request command + * + * @return command + */ + public Command convert2Command(){ + Command command = new Command(); + command.setType(CommandType.TASK_KILL_RESPONSE); + byte[] body = JsonSerializer.serialize(this); + command.setBody(body); + return command; + } + + @Override + public String toString() { + return "TaskKillResponseCommand{" + + "taskInstanceId=" + taskInstanceId + + ", host='" + host + '\'' + + ", status=" + status + + ", processId=" + processId + + ", appIds=" + appIds + + '}'; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java index 20cf8d9102..e4b21e2f89 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; @@ -56,7 +56,7 @@ public class GetLogBytesRequestCommand implements Serializable { public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.GET_LOG_BYTES_REQUEST); - byte[] body = FastJsonSerializer.serialize(this); + byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java index deaf9b8d85..349ec03855 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; @@ -57,7 +57,7 @@ public class GetLogBytesResponseCommand implements Serializable { public Command convert2Command(long opaque){ Command command = new Command(opaque); command.setType(CommandType.GET_LOG_BYTES_RESPONSE); - byte[] body = FastJsonSerializer.serialize(this); + byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java index 433c4a0b1f..00129c7e78 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; @@ -84,7 +84,7 @@ public class RollViewLogRequestCommand implements Serializable { public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.ROLL_VIEW_LOG_REQUEST); - byte[] body = FastJsonSerializer.serialize(this); + byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java index 591d787200..a4f4f86c9b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; @@ -57,7 +57,7 @@ public class RollViewLogResponseCommand implements Serializable { public Command convert2Command(long opaque){ Command command = new Command(opaque); command.setType(CommandType.ROLL_VIEW_LOG_RESPONSE); - byte[] body = FastJsonSerializer.serialize(this); + byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java index b4773d0c0d..1d51653eac 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; @@ -56,7 +56,7 @@ public class ViewLogRequestCommand implements Serializable { public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.VIEW_WHOLE_LOG_REQUEST); - byte[] body = FastJsonSerializer.serialize(this); + byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java index dffadade26..6940104f71 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; @@ -57,7 +57,7 @@ public class ViewLogResponseCommand implements Serializable { public Command convert2Command(long opaque){ Command command = new Command(opaque); command.setType(CommandType.VIEW_WHOLE_LOG_RESPONSE); - byte[] body = FastJsonSerializer.serialize(this); + byte[] body = JsonSerializer.serialize(this); command.setBody(body); return command; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/FastJsonSerializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java similarity index 56% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/FastJsonSerializer.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java index a18b8d5a7c..0c05232dd1 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/FastJsonSerializer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java @@ -16,13 +16,24 @@ */ package org.apache.dolphinscheduler.remote.utils; -import com.alibaba.fastjson.JSON; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; /** * json serialize or deserialize */ -public class FastJsonSerializer { +public class JsonSerializer { + private static final ObjectMapper objectMapper = new ObjectMapper(); + private static final Logger logger = LoggerFactory.getLogger(JsonSerializer.class); + + private JsonSerializer(){ + } /** * serialize to byte * @@ -31,7 +42,13 @@ public class FastJsonSerializer { * @return byte array */ public static byte[] serialize(T obj) { - String json = JSON.toJSONString(obj); + String json = ""; + try { + json = objectMapper.writeValueAsString(obj); + } catch (JsonProcessingException e) { + logger.error("serializeToString exception!", e); + } + return json.getBytes(Constants.UTF8); } @@ -42,7 +59,14 @@ public class FastJsonSerializer { * @return string */ public static String serializeToString(T obj) { - return JSON.toJSONString(obj); + String json = ""; + try { + json = objectMapper.writeValueAsString(obj); + } catch (JsonProcessingException e) { + logger.error("serializeToString exception!", e); + } + + return json; } /** @@ -54,7 +78,15 @@ public class FastJsonSerializer { * @return deserialize type */ public static T deserialize(byte[] src, Class clazz) { - return JSON.parseObject(src, clazz); + + String json = new String(src, StandardCharsets.UTF_8); + try { + return objectMapper.readValue(json, clazz); + } catch (IOException e) { + logger.error("deserialize exception!", e); + return null; + } + } } diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/FastJsonSerializerTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/JsonSerializerTest.java similarity index 84% rename from dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/FastJsonSerializerTest.java rename to dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/JsonSerializerTest.java index 97166cca70..cb92db7f25 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/FastJsonSerializerTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/JsonSerializerTest.java @@ -18,20 +18,20 @@ package org.apache.dolphinscheduler.remote; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.junit.Assert; import org.junit.Test; -public class FastJsonSerializerTest { +public class JsonSerializerTest { @Test public void testSerialize(){ TestObj testObj = new TestObj(); testObj.setAge(12); - byte[] serializeByte = FastJsonSerializer.serialize(testObj); + byte[] serializeByte = JsonSerializer.serialize(testObj); // - TestObj deserialize = FastJsonSerializer.deserialize(serializeByte, TestObj.class); + TestObj deserialize = JsonSerializer.deserialize(serializeByte, TestObj.class); Assert.assertEquals(testObj.getAge(), deserialize.getAge()); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java index 563f5c8459..7b4c721422 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java @@ -17,9 +17,10 @@ package org.apache.dolphinscheduler.server.entity; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; import java.util.Date; @@ -45,6 +46,7 @@ public class TaskExecutionContext implements Serializable{ /** * task start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** @@ -91,6 +93,7 @@ public class TaskExecutionContext implements Serializable{ /** * process instance schedule time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date scheduleTime; /** @@ -431,7 +434,7 @@ public class TaskExecutionContext implements Serializable{ public Command toCommand(){ TaskExecuteRequestCommand requestCommand = new TaskExecuteRequestCommand(); - requestCommand.setTaskExecutionContext(FastJsonSerializer.serializeToString(this)); + requestCommand.setTaskExecutionContext(JsonSerializer.serializeToString(this)); return requestCommand.convert2Command(); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java index 44ec68f89f..b7eea01f31 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java @@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.log.*; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,21 +61,21 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { final CommandType commandType = command.getType(); switch (commandType){ case GET_LOG_BYTES_REQUEST: - GetLogBytesRequestCommand getLogRequest = FastJsonSerializer.deserialize( + GetLogBytesRequestCommand getLogRequest = JsonSerializer.deserialize( command.getBody(), GetLogBytesRequestCommand.class); byte[] bytes = getFileContentBytes(getLogRequest.getPath()); GetLogBytesResponseCommand getLogResponse = new GetLogBytesResponseCommand(bytes); channel.writeAndFlush(getLogResponse.convert2Command(command.getOpaque())); break; case VIEW_WHOLE_LOG_REQUEST: - ViewLogRequestCommand viewLogRequest = FastJsonSerializer.deserialize( + ViewLogRequestCommand viewLogRequest = JsonSerializer.deserialize( command.getBody(), ViewLogRequestCommand.class); String msg = readWholeFileContent(viewLogRequest.getPath()); ViewLogResponseCommand viewLogResponse = new ViewLogResponseCommand(msg); channel.writeAndFlush(viewLogResponse.convert2Command(command.getOpaque())); break; case ROLL_VIEW_LOG_REQUEST: - RollViewLogRequestCommand rollViewLogRequest = FastJsonSerializer.deserialize( + RollViewLogRequestCommand rollViewLogRequest = JsonSerializer.deserialize( command.getBody(), RollViewLogRequestCommand.class); List lines = readPartFileContent(rollViewLogRequest.getPath(), rollViewLogRequest.getSkipLineNum(), rollViewLogRequest.getLimit()); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java index da2363fc81..d63b9252ba 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.consumer; -import com.alibaba.fastjson.JSONObject; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.SqoopJobType; import org.apache.dolphinscheduler.common.enums.TaskType; @@ -158,7 +157,7 @@ public class TaskPriorityQueueConsumer extends Thread{ TaskType taskType = TaskType.valueOf(taskInstance.getTaskType()); // task node - TaskNode taskNode = JSONObject.parseObject(taskInstance.getTaskJson(), TaskNode.class); + TaskNode taskNode = JSONUtils.parseObject(taskInstance.getTaskJson(), TaskNode.class); Integer userId = taskInstance.getProcessDefine() == null ? 0 : taskInstance.getProcessDefine().getUserId(); Tenant tenant = processService.getTenantForProcess(taskInstance.getProcessInstance().getTenantId(), userId); @@ -226,7 +225,7 @@ public class TaskPriorityQueueConsumer extends Thread{ * @param taskNode taskNode */ private void setProcedureTaskRelation(ProcedureTaskExecutionContext procedureTaskExecutionContext, TaskNode taskNode) { - ProcedureParameters procedureParameters = JSONObject.parseObject(taskNode.getParams(), ProcedureParameters.class); + ProcedureParameters procedureParameters = JSONUtils.parseObject(taskNode.getParams(), ProcedureParameters.class); int datasourceId = procedureParameters.getDatasource(); DataSource datasource = processService.findDataSourceById(datasourceId); procedureTaskExecutionContext.setConnectionParams(datasource.getConnectionParams()); @@ -238,7 +237,7 @@ public class TaskPriorityQueueConsumer extends Thread{ * @param taskNode taskNode */ private void setDataxTaskRelation(DataxTaskExecutionContext dataxTaskExecutionContext, TaskNode taskNode) { - DataxParameters dataxParameters = JSONObject.parseObject(taskNode.getParams(), DataxParameters.class); + DataxParameters dataxParameters = JSONUtils.parseObject(taskNode.getParams(), DataxParameters.class); DataSource dataSource = processService.findDataSourceById(dataxParameters.getDataSource()); DataSource dataTarget = processService.findDataSourceById(dataxParameters.getDataTarget()); @@ -264,7 +263,7 @@ public class TaskPriorityQueueConsumer extends Thread{ * @param taskNode taskNode */ private void setSqoopTaskRelation(SqoopTaskExecutionContext sqoopTaskExecutionContext, TaskNode taskNode) { - SqoopParameters sqoopParameters = JSONObject.parseObject(taskNode.getParams(), SqoopParameters.class); + SqoopParameters sqoopParameters = JSONUtils.parseObject(taskNode.getParams(), SqoopParameters.class); // sqoop job type is template set task relation if (sqoopParameters.getJobType().equals(SqoopJobType.TEMPLATE.getDescp())) { @@ -294,7 +293,7 @@ public class TaskPriorityQueueConsumer extends Thread{ * @param taskNode taskNode */ private void setSQLTaskRelation(SQLTaskExecutionContext sqlTaskExecutionContext, TaskNode taskNode) { - SqlParameters sqlParameters = JSONObject.parseObject(taskNode.getParams(), SqlParameters.class); + SqlParameters sqlParameters = JSONUtils.parseObject(taskNode.getParams(), SqlParameters.class); int datasourceId = sqlParameters.getDatasource(); DataSource datasource = processService.findDataSourceById(datasourceId); sqlTaskExecutionContext.setConnectionParams(datasource.getConnectionParams()); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java index 3460248dfb..e579535440 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java @@ -28,7 +28,7 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.remote.utils.ChannelUtils; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.apache.dolphinscheduler.server.master.cache.TaskInstanceCacheManager; import org.apache.dolphinscheduler.server.master.cache.impl.TaskInstanceCacheManagerImpl; import org.apache.dolphinscheduler.server.master.processor.queue.TaskResponseEvent; @@ -77,7 +77,7 @@ public class TaskAckProcessor implements NettyRequestProcessor { @Override public void process(Channel channel, Command command) { Preconditions.checkArgument(CommandType.TASK_EXECUTE_ACK == command.getType(), String.format("invalid command type : %s", command.getType())); - TaskExecuteAckCommand taskAckCommand = FastJsonSerializer.deserialize(command.getBody(), TaskExecuteAckCommand.class); + TaskExecuteAckCommand taskAckCommand = JsonSerializer.deserialize(command.getBody(), TaskExecuteAckCommand.class); logger.info("taskAckCommand : {}", taskAckCommand); taskInstanceCacheManager.cacheTaskInstance(taskAckCommand); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java index 3e8cdfdadc..2e51998cbd 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java @@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskKillResponseCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +45,7 @@ public class TaskKillResponseProcessor implements NettyRequestProcessor { public void process(Channel channel, Command command) { Preconditions.checkArgument(CommandType.TASK_KILL_RESPONSE == command.getType(), String.format("invalid command type : %s", command.getType())); - TaskKillResponseCommand responseCommand = FastJsonSerializer.deserialize(command.getBody(), TaskKillResponseCommand.class); + TaskKillResponseCommand responseCommand = JsonSerializer.deserialize(command.getBody(), TaskKillResponseCommand.class); logger.info("received task kill response command : {}", responseCommand); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java index 721b146d86..b04b930fd4 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java @@ -27,7 +27,7 @@ import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.apache.dolphinscheduler.server.master.cache.TaskInstanceCacheManager; import org.apache.dolphinscheduler.server.master.cache.impl.TaskInstanceCacheManagerImpl; import org.apache.dolphinscheduler.server.master.processor.queue.TaskResponseEvent; @@ -78,7 +78,7 @@ public class TaskResponseProcessor implements NettyRequestProcessor { public void process(Channel channel, Command command) { Preconditions.checkArgument(CommandType.TASK_EXECUTE_RESPONSE == command.getType(), String.format("invalid command type : %s", command.getType())); - TaskExecuteResponseCommand responseCommand = FastJsonSerializer.deserialize(command.getBody(), TaskExecuteResponseCommand.class); + TaskExecuteResponseCommand responseCommand = JsonSerializer.deserialize(command.getBody(), TaskExecuteResponseCommand.class); logger.info("received command : {}", responseCommand); taskInstanceCacheManager.cacheTaskInstance(responseCommand); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java index 9e8813fd7f..051cc388bf 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.master.processor.queue; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import java.util.Date; @@ -44,11 +45,13 @@ public class TaskResponseEvent { /** * start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date startTime; /** * end time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date endTime; /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java index 7e3950df1f..61fc27f49a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java @@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.common.model.DependentItem; import org.apache.dolphinscheduler.common.model.DependentTaskModel; import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; import org.apache.dolphinscheduler.common.utils.DependentUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java index 015c20024c..7c8da5ab8c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.server.master.runner; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DependResult; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; @@ -23,7 +24,7 @@ import org.apache.dolphinscheduler.common.model.DependentTaskModel; import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.DependentUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -53,6 +54,7 @@ public class DependentTaskExecThread extends MasterBaseTaskExecThread { /** * dependent date */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date dependentDate; /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java index 3226d82304..c903dbc6b0 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java @@ -20,7 +20,7 @@ import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.sift.SiftingAppender; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -191,7 +191,7 @@ public class MasterBaseTaskExecThread implements Callable { return true; }catch (Exception e){ logger.error("submit task Exception: ", e); - logger.error("task error : %s", JSONUtils.toJson(taskInstance)); + logger.error("task error : %s", JSONUtils.toJsonString(taskInstance)); return false; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index e0110adfcd..211da1a433 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.server.master.runner; -import com.alibaba.fastjson.JSON; import com.google.common.collect.Lists; import org.apache.commons.io.FileUtils; import org.apache.dolphinscheduler.common.Constants; @@ -295,7 +294,7 @@ public class MasterExecThread implements Runnable { processInstance.setScheduleTime(scheduleDate); if(cmdParam.containsKey(Constants.CMDPARAM_RECOVERY_START_NODE_STRING)){ cmdParam.remove(Constants.CMDPARAM_RECOVERY_START_NODE_STRING); - processInstance.setCommandParam(JSONUtils.toJson(cmdParam)); + processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam)); } List taskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId()); @@ -467,7 +466,7 @@ public class MasterExecThread implements Runnable { // process instance id taskInstance.setProcessInstanceId(processInstance.getId()); // task instance node json - taskInstance.setTaskJson(JSON.toJSONString(taskNode)); + taskInstance.setTaskJson(JSONUtils.toJsonString(taskNode)); // task instance type taskInstance.setTaskType(taskNode.getType()); // task instance whether alert diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java index 105584fe99..ed491507e3 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.server.master.runner; - -import com.alibaba.fastjson.JSON; - import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; @@ -38,10 +35,10 @@ import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType; import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager; import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; -import org.springframework.beans.factory.annotation.Autowired; import java.util.Date; import java.util.Set; +import org.apache.dolphinscheduler.common.utils.*; /** @@ -239,7 +236,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { */ private TaskTimeoutParameter getTaskTimeoutParameter(){ String taskJson = taskInstance.getTaskJson(); - TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class); + TaskNode taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); return taskNode.getTaskTimeoutParameter(); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java index 050af1eec5..49ec9d3fdd 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java @@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.DaoFactory; import org.apache.dolphinscheduler.dao.entity.Alert; @@ -145,7 +145,7 @@ public class AlertManager { failedTaskMap.put("log path", task.getLogPath()); failedTaskList.add(failedTaskMap); } - res = JSONUtils.toJson(failedTaskList); + res = JSONUtils.toJsonString(failedTaskList); } return res; @@ -170,7 +170,7 @@ public class AlertManager { toleranceWorkerContentMap.put("task retry times", String.valueOf(taskInstance.getRetryTimes())); toleranceTaskInstanceList.add(toleranceWorkerContentMap); } - return JSONUtils.toJson(toleranceTaskInstanceList); + return JSONUtils.toJsonString(toleranceTaskInstanceList); } /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java index ae26b8d7dd..50f8989b0e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java @@ -19,25 +19,21 @@ package org.apache.dolphinscheduler.server.worker.processor; import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.sift.SiftingAppender; -import com.alibaba.fastjson.JSONObject; import com.github.rholder.retry.RetryException; import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.TaskType; -import org.apache.dolphinscheduler.common.utils.OSUtils; -import org.apache.dolphinscheduler.common.utils.RetryerUtils; -import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.FileUtils; -import org.apache.dolphinscheduler.common.utils.Preconditions; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; +import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.runner.TaskExecuteThread; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; @@ -82,14 +78,14 @@ public class TaskExecuteProcessor implements NettyRequestProcessor { Preconditions.checkArgument(CommandType.TASK_EXECUTE_REQUEST == command.getType(), String.format("invalid command type : %s", command.getType())); - TaskExecuteRequestCommand taskRequestCommand = FastJsonSerializer.deserialize( + TaskExecuteRequestCommand taskRequestCommand = JsonSerializer.deserialize( command.getBody(), TaskExecuteRequestCommand.class); logger.info("received command : {}", taskRequestCommand); String contextJson = taskRequestCommand.getTaskExecutionContext(); - TaskExecutionContext taskExecutionContext = JSONObject.parseObject(contextJson, TaskExecutionContext.class); + TaskExecutionContext taskExecutionContext = JSONUtils.parseObject(contextJson, TaskExecutionContext.class); taskExecutionContext.setHost(OSUtils.getHost() + ":" + workerConfig.getListenPort()); // local execute path diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java index b6f58279b1..17ae6a43b1 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java @@ -29,7 +29,7 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskKillRequestCommand; import org.apache.dolphinscheduler.remote.command.TaskKillResponseCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.Pair; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; @@ -83,7 +83,7 @@ public class TaskKillProcessor implements NettyRequestProcessor { @Override public void process(Channel channel, Command command) { Preconditions.checkArgument(CommandType.TASK_KILL_REQUEST == command.getType(), String.format("invalid command type : %s", command.getType())); - TaskKillRequestCommand killCommand = FastJsonSerializer.deserialize(command.getBody(), TaskKillRequestCommand.class); + TaskKillRequestCommand killCommand = JsonSerializer.deserialize(command.getBody(), TaskKillRequestCommand.class); logger.info("received kill command : {}", killCommand); Pair> result = doKill(killCommand); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java index d314c5535d..b6ab894020 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java @@ -16,15 +16,14 @@ */ package org.apache.dolphinscheduler.server.worker.runner; +import org.apache.dolphinscheduler.common.utils.*; -import com.alibaba.fastjson.JSONObject; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService; @@ -80,7 +79,7 @@ public class TaskExecuteThread implements Runnable { try { logger.info("script path : {}", taskExecutionContext.getExecutePath()); // task node - TaskNode taskNode = JSONObject.parseObject(taskExecutionContext.getTaskJson(), TaskNode.class); + TaskNode taskNode = JSONUtils.parseObject(taskExecutionContext.getTaskJson(), TaskNode.class); // copy hdfs/minio file to local downloadResource(taskExecutionContext.getExecutePath(), @@ -152,7 +151,7 @@ public class TaskExecuteThread implements Runnable { // global params string String globalParamsStr = taskExecutionContext.getGlobalParams(); if (globalParamsStr != null) { - List globalParamsList = JSONObject.parseArray(globalParamsStr, Property.class); + List globalParamsList = JSONUtils.toList(globalParamsStr, Property.class); globalParamsMap.putAll(globalParamsList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue))); } return globalParamsMap; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java index 36b974b97a..ae03932a52 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java @@ -30,7 +30,7 @@ import org.apache.dolphinscheduler.common.task.shell.ShellParameters; import org.apache.dolphinscheduler.common.task.spark.SparkParameters; import org.apache.dolphinscheduler.common.task.sql.SqlParameters; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.TaskRecordDao; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.utils.ParamUtils; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java index 00e78d37d1..0e619eb306 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.server.worker.task; +import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.Direct; @@ -80,6 +81,7 @@ public class TaskProps { /** * task start time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date taskStartTime; /** @@ -99,6 +101,7 @@ public class TaskProps { /** * schedule time */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") private Date scheduleTime; /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index 218906d91c..a14a5d64f5 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -17,41 +17,26 @@ package org.apache.dolphinscheduler.server.worker.task.datax; -import java.io.File; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; - +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; +import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; +import com.alibaba.druid.sql.ast.statement.*; +import com.alibaba.druid.sql.parser.SQLStatementParser; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.io.FileUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; -import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.datax.DataxParameters; import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; -import org.apache.dolphinscheduler.dao.entity.DataSource; -import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.server.entity.DataxTaskExecutionContext; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.utils.DataxUtils; @@ -59,20 +44,18 @@ import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.CommandExecuteResult; import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; -import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; -import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; -import com.alibaba.druid.sql.ast.SQLStatement; -import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; -import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; -import com.alibaba.druid.sql.ast.statement.SQLSelect; -import com.alibaba.druid.sql.ast.statement.SQLSelectItem; -import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock; -import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; -import com.alibaba.druid.sql.ast.statement.SQLUnionQuery; -import com.alibaba.druid.sql.parser.SQLStatementParser; -import com.alibaba.fastjson.JSONObject; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.sql.*; +import java.util.*; /** @@ -173,19 +156,19 @@ public class DataxTask extends AbstractTask { */ @Override public void cancelApplication(boolean cancelApplication) - throws Exception { + throws Exception { // cancel process shellCommandExecutor.cancelApplication(); } /** * build datax configuration file - * + * * @return datax json file name * @throws Exception if error throws Exception */ private String buildDataxJsonFile() - throws Exception { + throws Exception { // generate json String fileName = String.format("%s/%s_job.json", taskExecutionContext.getExecutePath(), @@ -216,14 +199,14 @@ public class DataxTask extends AbstractTask { } }else { + ObjectNode job = JSONUtils.createObjectNode(); + job.putArray("content").addAll(buildDataxJobContentJson()); + job.set("setting", buildDataxJobSettingJson()); - JSONObject job = new JSONObject(); - job.put("content", buildDataxJobContentJson()); - job.put("setting", buildDataxJobSettingJson()); + ObjectNode root = JSONUtils.createObjectNode(); - JSONObject root = new JSONObject(); - root.put("job", job); - root.put("core", buildDataxCoreJson()); + root.set("job", job); + root.set("core", buildDataxCoreJson()); json = root.toString(); } @@ -236,13 +219,13 @@ public class DataxTask extends AbstractTask { /** * build datax job config - * + * * @return collection of datax job config JSONObject * @throws SQLException if error throws SQLException */ - private List buildDataxJobContentJson() throws SQLException { - DataxTaskExecutionContext dataxTaskExecutionContext = taskExecutionContext.getDataxTaskExecutionContext(); + private List buildDataxJobContentJson() { + DataxTaskExecutionContext dataxTaskExecutionContext = taskExecutionContext.getDataxTaskExecutionContext(); BaseDataSource dataSourceCfg = DataSourceFactory.getDatasource(DbType.of(dataxTaskExecutionContext.getSourcetype()), dataxTaskExecutionContext.getSourceConnectionParams()); @@ -250,52 +233,79 @@ public class DataxTask extends AbstractTask { BaseDataSource dataTargetCfg = DataSourceFactory.getDatasource(DbType.of(dataxTaskExecutionContext.getTargetType()), dataxTaskExecutionContext.getTargetConnectionParams()); - List readerConnArr = new ArrayList<>(); - JSONObject readerConn = new JSONObject(); - readerConn.put("querySql", new String[] {dataXParameters.getSql()}); - readerConn.put("jdbcUrl", new String[] {dataSourceCfg.getJdbcUrl()}); + List readerConnArr = new ArrayList<>(); + ObjectNode readerConn = JSONUtils.createObjectNode(); + + ArrayNode sqlArr = readerConn.putArray("querySql"); + for (String sql : new String[]{dataXParameters.getSql()}) { + sqlArr.add(sql); + } + + ArrayNode urlArr = readerConn.putArray("jdbcUrl"); + for (String url : new String[]{dataSourceCfg.getJdbcUrl()}) { + urlArr.add(url); + } + readerConnArr.add(readerConn); - JSONObject readerParam = new JSONObject(); + ObjectNode readerParam = JSONUtils.createObjectNode(); readerParam.put("username", dataSourceCfg.getUser()); readerParam.put("password", dataSourceCfg.getPassword()); - readerParam.put("connection", readerConnArr); + readerParam.putArray("connection").addAll(readerConnArr); + - JSONObject reader = new JSONObject(); + ObjectNode reader = JSONUtils.createObjectNode(); reader.put("name", DataxUtils.getReaderPluginName(DbType.of(dataxTaskExecutionContext.getSourcetype()))); - reader.put("parameter", readerParam); + reader.set("parameter", readerParam); + + List writerConnArr = new ArrayList<>(); + ObjectNode writerConn = JSONUtils.createObjectNode(); + ArrayNode tableArr = writerConn.putArray("table"); + for (String table : new String[]{dataXParameters.getTargetTable()}) { + tableArr.add(table); + } - List writerConnArr = new ArrayList<>(); - JSONObject writerConn = new JSONObject(); - writerConn.put("table", new String[] {dataXParameters.getTargetTable()}); writerConn.put("jdbcUrl", dataTargetCfg.getJdbcUrl()); writerConnArr.add(writerConn); - JSONObject writerParam = new JSONObject(); + ObjectNode writerParam = JSONUtils.createObjectNode(); writerParam.put("username", dataTargetCfg.getUser()); writerParam.put("password", dataTargetCfg.getPassword()); - writerParam.put("column", - parsingSqlColumnNames(DbType.of(dataxTaskExecutionContext.getSourcetype()), - DbType.of(dataxTaskExecutionContext.getTargetType()), - dataSourceCfg, dataXParameters.getSql())); - writerParam.put("connection", writerConnArr); + + String[] columns = parsingSqlColumnNames(DbType.of(dataxTaskExecutionContext.getSourcetype()), + DbType.of(dataxTaskExecutionContext.getTargetType()), + dataSourceCfg, dataXParameters.getSql()); + + ArrayNode columnArr = writerParam.putArray("column"); + for (String column : columns) { + columnArr.add(column); + } + writerParam.putArray("connection").addAll(writerConnArr); + if (CollectionUtils.isNotEmpty(dataXParameters.getPreStatements())) { - writerParam.put("preSql", dataXParameters.getPreStatements()); + ArrayNode preSqlArr = writerParam.putArray("preSql"); + for (String preSql : dataXParameters.getPreStatements()) { + preSqlArr.add(preSql); + } + } if (CollectionUtils.isNotEmpty(dataXParameters.getPostStatements())) { - writerParam.put("postSql", dataXParameters.getPostStatements()); + ArrayNode postSqlArr = writerParam.putArray("postSql"); + for (String postSql : dataXParameters.getPostStatements()) { + postSqlArr.add(postSql); + } } - JSONObject writer = new JSONObject(); + ObjectNode writer = JSONUtils.createObjectNode(); writer.put("name", DataxUtils.getWriterPluginName(DbType.of(dataxTaskExecutionContext.getTargetType()))); - writer.put("parameter", writerParam); + writer.set("parameter", writerParam); - List contentList = new ArrayList<>(); - JSONObject content = new JSONObject(); - content.put("reader", reader); - content.put("writer", writer); + List contentList = new ArrayList<>(); + ObjectNode content = JSONUtils.createObjectNode(); + content.put("reader", reader.toString()); + content.put("writer", writer.toString()); contentList.add(content); return contentList; @@ -303,11 +313,13 @@ public class DataxTask extends AbstractTask { /** * build datax setting config - * + * * @return datax setting config JSONObject */ - private JSONObject buildDataxJobSettingJson() { - JSONObject speed = new JSONObject(); + private ObjectNode buildDataxJobSettingJson() { + + ObjectNode speed = JSONUtils.createObjectNode(); + speed.put("channel", DATAX_CHANNEL_COUNT); if (dataXParameters.getJobSpeedByte() > 0) { @@ -318,19 +330,20 @@ public class DataxTask extends AbstractTask { speed.put("record", dataXParameters.getJobSpeedRecord()); } - JSONObject errorLimit = new JSONObject(); + ObjectNode errorLimit = JSONUtils.createObjectNode(); errorLimit.put("record", 0); errorLimit.put("percentage", 0); - JSONObject setting = new JSONObject(); - setting.put("speed", speed); - setting.put("errorLimit", errorLimit); + ObjectNode setting = JSONUtils.createObjectNode(); + setting.put("speed", speed.toString()); + setting.put("errorLimit", errorLimit.toString()); return setting; } - private JSONObject buildDataxCoreJson() { - JSONObject speed = new JSONObject(); + private ObjectNode buildDataxCoreJson() { + + ObjectNode speed = JSONUtils.createObjectNode(); speed.put("channel", DATAX_CHANNEL_COUNT); if (dataXParameters.getJobSpeedByte() > 0) { @@ -341,26 +354,26 @@ public class DataxTask extends AbstractTask { speed.put("record", dataXParameters.getJobSpeedRecord()); } - JSONObject channel = new JSONObject(); - channel.put("speed", speed); + ObjectNode channel = JSONUtils.createObjectNode(); + channel.set("speed", speed); - JSONObject transport = new JSONObject(); - transport.put("channel", channel); + ObjectNode transport = JSONUtils.createObjectNode(); + transport.set("channel", channel); - JSONObject core = new JSONObject(); - core.put("transport", transport); + ObjectNode core = JSONUtils.createObjectNode(); + core.set("transport", transport); return core; } /** * create command - * + * * @return shell command file name * @throws Exception if error throws Exception */ private String buildShellCommandFile(String jobConfigFilePath) - throws Exception { + throws Exception { // generate scripts String fileName = String.format("%s/%s_node.sh", taskExecutionContext.getExecutePath(), @@ -411,7 +424,7 @@ public class DataxTask extends AbstractTask { /** * parsing synchronized column names in SQL statements - * + * * @param dsType * the database type of the data source * @param dtType @@ -437,7 +450,7 @@ public class DataxTask extends AbstractTask { /** * try grammatical parsing column - * + * * @param dbType * database type * @param sql @@ -467,7 +480,7 @@ public class DataxTask extends AbstractTask { } notNull(selectItemList, - String.format("select query type [%s] is not support", sqlSelect.getQuery().toString())); + String.format("select query type [%s] is not support", sqlSelect.getQuery().toString())); columnNames = new String[selectItemList.size()]; for (int i = 0; i < selectItemList.size(); i++ ) { @@ -487,12 +500,12 @@ public class DataxTask extends AbstractTask { } } else { throw new RuntimeException( - String.format("grammatical analysis sql column [ %s ] failed", item.toString())); + String.format("grammatical analysis sql column [ %s ] failed", item.toString())); } if (columnName == null) { throw new RuntimeException( - String.format("grammatical analysis sql column [ %s ] failed", item.toString())); + String.format("grammatical analysis sql column [ %s ] failed", item.toString())); } columnNames[i] = columnName; @@ -508,7 +521,7 @@ public class DataxTask extends AbstractTask { /** * try to execute sql to resolve column names - * + * * @param baseDataSource * the database connection parameters * @param sql @@ -521,10 +534,10 @@ public class DataxTask extends AbstractTask { sql = sql.replace(";", ""); try ( - Connection connection = DriverManager.getConnection(baseDataSource.getJdbcUrl(), baseDataSource.getUser(), - baseDataSource.getPassword()); - PreparedStatement stmt = connection.prepareStatement(sql); - ResultSet resultSet = stmt.executeQuery()) { + Connection connection = DriverManager.getConnection(baseDataSource.getJdbcUrl(), baseDataSource.getUser(), + baseDataSource.getPassword()); + PreparedStatement stmt = connection.prepareStatement(sql); + ResultSet resultSet = stmt.executeQuery()) { ResultSetMetaData md = resultSet.getMetaData(); int num = md.getColumnCount(); @@ -552,4 +565,4 @@ public class DataxTask extends AbstractTask { } } -} +} \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java index c377d5fa68..58d5eafb2d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java @@ -21,7 +21,7 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java index 130137da29..87adaab9af 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java @@ -17,8 +17,7 @@ package org.apache.dolphinscheduler.server.worker.task.http; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.io.Charsets; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; @@ -28,10 +27,7 @@ import org.apache.dolphinscheduler.common.process.HttpProperty; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.http.HttpParameters; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; @@ -51,7 +47,6 @@ import org.slf4j.Logger; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -95,7 +90,7 @@ public class HttpTask extends AbstractTask { @Override public void init() { logger.info("http task params {}", taskExecutionContext.getTaskParams()); - this.httpParameters = JSONObject.parseObject(taskExecutionContext.getTaskParams(), HttpParameters.class); + this.httpParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), HttpParameters.class); if (!httpParameters.checkParameters()) { throw new RuntimeException("http task params is not valid"); @@ -145,10 +140,10 @@ public class HttpTask extends AbstractTask { List httpPropertyList = new ArrayList<>(); if(CollectionUtils.isNotEmpty(httpParameters.getHttpParams() )){ for (HttpProperty httpProperty: httpParameters.getHttpParams()) { - String jsonObject = JSON.toJSONString(httpProperty); + String jsonObject = JSONUtils.toJsonString(httpProperty); String params = ParameterUtils.convertParameterPlaceholders(jsonObject,ParamUtils.convert(paramsMap)); logger.info("http request params：{}",params); - httpPropertyList.add(JSON.parseObject(params,HttpProperty.class)); + httpPropertyList.add(JSONUtils.parseObject(params,HttpProperty.class)); } } addRequestParams(builder,httpPropertyList); @@ -248,7 +243,7 @@ public class HttpTask extends AbstractTask { */ protected void addRequestParams(RequestBuilder builder,List httpPropertyList) { if(CollectionUtils.isNotEmpty(httpPropertyList)){ - JSONObject jsonParam = new JSONObject(); + ObjectNode jsonParam = JSONUtils.createObjectNode(); for (HttpProperty property: httpPropertyList){ if(property.getHttpParametersType() != null){ if (property.getHttpParametersType().equals(HttpParametersType.PARAMETER)){ diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java index fed7b27739..3a8f75a64f 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java @@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.mr.MapreduceParameters; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java index 72d5616e5b..a11a8c0d45 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java @@ -16,8 +16,6 @@ */ package org.apache.dolphinscheduler.server.worker.task.processdure; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; import com.cronutils.utils.StringUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.*; @@ -25,6 +23,7 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.procedure.ProcedureParameters; import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; @@ -74,7 +73,7 @@ public class ProcedureTask extends AbstractTask { logger.info("procedure task params {}", taskExecutionContext.getTaskParams()); - this.procedureParameters = JSONObject.parseObject(taskExecutionContext.getTaskParams(), ProcedureParameters.class); + this.procedureParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), ProcedureParameters.class); // check parameters diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java index 7a66227b8d..367da80a0c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java @@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.python.PythonParameters; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.utils.ParamUtils; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java index f24aa54ffb..d846d5f83e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java @@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.shell.ShellParameters; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTask.java index 505d88fb37..c540d27810 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTask.java @@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.spark.SparkParameters; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index ccd4bb2214..d201a1f261 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -16,9 +16,8 @@ */ package org.apache.dolphinscheduler.server.worker.task.sql; -import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; -import com.alibaba.fastjson.serializer.SerializerFeature; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.alert.utils.MailUtils; import org.apache.dolphinscheduler.common.Constants; @@ -86,7 +85,7 @@ public class SqlTask extends AbstractTask { this.taskExecutionContext = taskExecutionContext; logger.info("sql task params {}", taskExecutionContext.getTaskParams()); - this.sqlParameters = JSONObject.parseObject(taskExecutionContext.getTaskParams(), SqlParameters.class); + this.sqlParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), SqlParameters.class); if (!sqlParameters.checkParameters()) { throw new RuntimeException("sql task params is not valid"); @@ -254,30 +253,31 @@ public class SqlTask extends AbstractTask { * @throws Exception */ private void resultProcess(ResultSet resultSet) throws Exception{ - JSONArray resultJSONArray = new JSONArray(); + ArrayNode resultJSONArray = JSONUtils.createArrayNode(); ResultSetMetaData md = resultSet.getMetaData(); int num = md.getColumnCount(); int rowCount = 0; while (rowCount < LIMIT && resultSet.next()) { - JSONObject mapOfColValues = new JSONObject(true); + ObjectNode mapOfColValues = JSONUtils.createObjectNode(); for (int i = 1; i <= num; i++) { - mapOfColValues.put(md.getColumnName(i), resultSet.getObject(i)); + mapOfColValues.set(md.getColumnName(i), JSONUtils.toJsonNode(resultSet.getObject(i))); } resultJSONArray.add(mapOfColValues); rowCount++; } - logger.debug("execute sql : {}", JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); + String result = JSONUtils.toJsonString(resultJSONArray); + logger.debug("execute sql : {}", result); // if there is a result set - if (!resultJSONArray.isEmpty() ) { + if (!resultJSONArray.isEmpty(null) ) { if (StringUtils.isNotEmpty(sqlParameters.getTitle())) { sendAttachment(sqlParameters.getTitle(), - JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); + JSONUtils.toJsonString(resultJSONArray)); }else{ sendAttachment(taskExecutionContext.getTaskName() + " query resultsets ", - JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); + JSONUtils.toJsonString(resultJSONArray)); } } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java index 9f54d089be..c66fe95592 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.server.worker.task.sqoop; -import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; @@ -27,7 +26,9 @@ import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractYarnTask; import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.SqoopJobGenerator; import org.slf4j.Logger; + import java.util.Map; +import org.apache.dolphinscheduler.common.utils.*; /** * sqoop task extends the shell task @@ -50,7 +51,7 @@ public class SqoopTask extends AbstractYarnTask { public void init() throws Exception { logger.info("sqoop task params {}", taskExecutionContext.getTaskParams()); sqoopParameters = - JSON.parseObject(taskExecutionContext.getTaskParams(),SqoopParameters.class); + JSONUtils.parseObject(taskExecutionContext.getTaskParams(),SqoopParameters.class); if (!sqoopParameters.checkParameters()) { throw new RuntimeException("sqoop task params is not valid"); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HdfsSourceGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HdfsSourceGenerator.java index 41e56682ae..ded0d6db17 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HdfsSourceGenerator.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HdfsSourceGenerator.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.sources; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHdfsParameter; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ISourceGenerator; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HiveSourceGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HiveSourceGenerator.java index ea12616825..131b616025 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HiveSourceGenerator.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HiveSourceGenerator.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.sources; import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHiveParameter; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ISourceGenerator; import org.slf4j.Logger; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java index 402f4f8ee0..f4d0a67221 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java @@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.common.enums.QueryType; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceMysqlParameter; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; import org.apache.dolphinscheduler.server.entity.SqoopTaskExecutionContext; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HdfsTargetGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HdfsTargetGenerator.java index 64ea75e742..f94d10a6d1 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HdfsTargetGenerator.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HdfsTargetGenerator.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.targets; import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHdfsParameter; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ITargetGenerator; import org.slf4j.Logger; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java index dc5440b529..83c4123c45 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.targets; import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHiveParameter; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ITargetGenerator; import org.slf4j.Logger; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java index 5b0b75d8e2..406c6f661d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java @@ -20,7 +20,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetMysqlParameter; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; import org.apache.dolphinscheduler.dao.entity.DataSource; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java index 4dbf9df70e..5875d21758 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.master; -import com.alibaba.fastjson.JSON; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.enums.*; import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.utils.DateUtils; @@ -85,7 +85,7 @@ public class MasterExecThreadTest { Map cmdParam = new HashMap<>(); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, "2020-01-01 00:00:00"); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, "2020-01-31 23:00:00"); - Mockito.when(processInstance.getCommandParam()).thenReturn(JSON.toJSONString(cmdParam)); + Mockito.when(processInstance.getCommandParam()).thenReturn(JSONUtils.toJsonString(cmdParam)); ProcessDefinition processDefinition = new ProcessDefinition(); processDefinition.setGlobalParamMap(Collections.EMPTY_MAP); processDefinition.setGlobalParamList(Collections.EMPTY_LIST); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java index 46c68fb044..48b34d5f88 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java @@ -23,7 +23,8 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; import org.apache.dolphinscheduler.server.utils.ParamUtils; -import com.alibaba.fastjson.JSON; +import org.apache.dolphinscheduler.common.utils.*; + import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -95,7 +96,7 @@ public class ParamsTest { Map paramsMap = ParamUtils.convert(globalParams, globalParamsMap, localParams, CommandType.START_PROCESS, new Date()); - logger.info(JSON.toJSONString(paramsMap)); + logger.info(JSONUtils.toJsonString(paramsMap)); } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java index 60a5259f44..220cce5d18 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java @@ -17,11 +17,11 @@ package org.apache.dolphinscheduler.server.utils; -import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.Direct; import org.apache.dolphinscheduler.common.process.Property; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; @@ -80,11 +80,9 @@ public class ParamUtilsTest { public void testConvert() { //The expected value - String expected = "{\"global_param\":{\"direct\":\"IN\",\"prop\":\"global_param\",\"type\":\"VARCHAR\",\"value\":\"20191229\"},\"local_param\":{\"direct\":\"IN\",\"prop\":\"local_param\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}}"; - + String expected = "{\"global_param\":{\"prop\":\"global_param\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"20191229\"},\"local_param\":{\"prop\":\"local_param\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}}"; //The expected value when globalParams is null but localParams is not null - String expected1 = "{\"local_param\":{\"direct\":\"IN\",\"prop\":\"local_param\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}}"; - + String expected1 = "{\"local_param\":{\"prop\":\"local_param\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}}"; //Define expected date , the month is 0-base Calendar calendar = Calendar.getInstance(); calendar.set(2019,11,30); @@ -92,7 +90,7 @@ public class ParamUtilsTest { //Invoke convert Map paramsMap = ParamUtils.convert(globalParams, globalParamsMap, localParams, CommandType.START_PROCESS, date); - String result = JSON.toJSONString(paramsMap); + String result = JSONUtils.toJsonString(paramsMap); assertEquals(expected, result); for (Map.Entry entry : paramsMap.entrySet()) { @@ -104,7 +102,7 @@ public class ParamUtilsTest { //Invoke convert with null globalParams Map paramsMap1 = ParamUtils.convert(null, globalParamsMap, localParams, CommandType.START_PROCESS, date); - String result1 = JSON.toJSONString(paramsMap1); + String result1 = JSONUtils.toJsonString(paramsMap1); assertEquals(expected1, result1); //Null check, invoke convert with null globalParams and null localParams @@ -123,7 +121,7 @@ public class ParamUtilsTest { //Invoke convert Map paramsMap = ParamUtils.convert(globalParams); - String result = JSON.toJSONString(paramsMap); + String result = JSONUtils.toJsonString(paramsMap); assertEquals(expected, result); logger.info(result); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index acc7a22ff0..ed3479ee0d 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -16,14 +16,12 @@ */ package org.apache.dolphinscheduler.server.worker.shell; -import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; -import org.apache.dolphinscheduler.server.worker.task.TaskManager; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -32,6 +30,7 @@ import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.dolphinscheduler.common.utils.*; import java.util.Date; @@ -68,7 +67,7 @@ public class ShellCommandExecutorTest { TaskInstance taskInstance = processService.findTaskInstanceById(7657); String taskJson = taskInstance.getTaskJson(); - TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class); + TaskNode taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); taskProps.setTaskParams(taskNode.getParams()); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java index 49301c3906..dbaa13215f 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java @@ -16,18 +16,17 @@ */ package org.apache.dolphinscheduler.server.worker.sql; -import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; -import org.apache.dolphinscheduler.server.worker.task.TaskManager; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.dolphinscheduler.common.utils.*; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -112,7 +111,7 @@ public class SqlExecutorTest { TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); String taskJson = taskInstance.getTaskJson(); - TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class); + TaskNode taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); taskProps.setTaskParams(taskNode.getParams()); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java index a2a46ef5a5..6c3cad5671 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java @@ -21,10 +21,12 @@ import java.lang.reflect.Method; import java.util.Arrays; import java.util.Date; import java.util.List; +import java.util.UUID; -import com.alibaba.fastjson.JSONObject; -import org.apache.dolphinscheduler.common.enums.CommandType; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; import org.apache.dolphinscheduler.dao.entity.DataSource; @@ -70,7 +72,12 @@ public class DataxTaskTest { @Before public void before() - throws Exception { + throws Exception { + setTaskParems(0); + } + + private void setTaskParems(Integer customConfig) { + processService = Mockito.mock(ProcessService.class); shellCommandExecutor = Mockito.mock(ShellCommandExecutor.class); @@ -87,13 +94,20 @@ public class DataxTaskTest { props.setEnvFile(".dolphinscheduler_env.sh"); props.setTaskStartTime(new Date()); props.setTaskTimeout(0); - props.setTaskParams( - "{\"targetTable\":\"test\",\"postStatements\":[],\"jobSpeedByte\":1024,\"jobSpeedRecord\":1000,\"dtType\":\"MYSQL\",\"datasource\":1,\"dsType\":\"MYSQL\",\"datatarget\":2,\"jobSpeedByte\":0,\"sql\":\"select 1 as test from dual\",\"preStatements\":[\"delete from test\"],\"postStatements\":[\"delete from test\"]}"); + if (customConfig == 1) { + props.setTaskParams( + "{\"customConfig\":1, \"localParams\":[{\"prop\":\"test\",\"value\":\"38294729\"}],\"json\":\"{\\\"job\\\":{\\\"setting\\\":{\\\"speed\\\":{\\\"byte\\\":1048576},\\\"errorLimit\\\":{\\\"record\\\":0,\\\"percentage\\\":0.02}},\\\"content\\\":[{\\\"reader\\\":{\\\"name\\\":\\\"rdbmsreader\\\",\\\"parameter\\\":{\\\"username\\\":\\\"xxx\\\",\\\"password\\\":\\\"${test}\\\",\\\"column\\\":[\\\"id\\\",\\\"name\\\"],\\\"splitPk\\\":\\\"pk\\\",\\\"connection\\\":[{\\\"querySql\\\":[\\\"SELECT * from dual\\\"],\\\"jdbcUrl\\\":[\\\"jdbc:dm://ip:port/database\\\"]}],\\\"fetchSize\\\":1024,\\\"where\\\":\\\"1 = 1\\\"}},\\\"writer\\\":{\\\"name\\\":\\\"streamwriter\\\",\\\"parameter\\\":{\\\"print\\\":true}}}]}}\"}"); + +// "{\"customConfig\":1,\"json\":\"{\\\"job\\\":{\\\"setting\\\":{\\\"speed\\\":{\\\"byte\\\":1048576},\\\"errorLimit\\\":{\\\"record\\\":0,\\\"percentage\\\":0.02}},\\\"content\\\":[{\\\"reader\\\":{\\\"name\\\":\\\"rdbmsreader\\\",\\\"parameter\\\":{\\\"username\\\":\\\"xxx\\\",\\\"password\\\":\\\"xxx\\\",\\\"column\\\":[\\\"id\\\",\\\"name\\\"],\\\"splitPk\\\":\\\"pk\\\",\\\"connection\\\":[{\\\"querySql\\\":[\\\"SELECT * from dual\\\"],\\\"jdbcUrl\\\":[\\\"jdbc:dm://ip:port/database\\\"]}],\\\"fetchSize\\\":1024,\\\"where\\\":\\\"1 = 1\\\"}},\\\"writer\\\":{\\\"name\\\":\\\"streamwriter\\\",\\\"parameter\\\":{\\\"print\\\":true}}}]}}\"}"); + } else { + props.setTaskParams( + "{\"customConfig\":0,\"targetTable\":\"test\",\"postStatements\":[],\"jobSpeedByte\":1024,\"jobSpeedRecord\":1000,\"dtType\":\"MYSQL\",\"dataSource\":1,\"dsType\":\"MYSQL\",\"dataTarget\":2,\"jobSpeedByte\":0,\"sql\":\"select 1 as test from dual\",\"preStatements\":[\"delete from test\"],\"postStatements\":[\"delete from test\"]}"); + } taskExecutionContext = Mockito.mock(TaskExecutionContext.class); Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(props.getTaskParams()); Mockito.when(taskExecutionContext.getExecutePath()).thenReturn("/tmp"); - Mockito.when(taskExecutionContext.getTaskAppId()).thenReturn("1"); + Mockito.when(taskExecutionContext.getTaskAppId()).thenReturn(UUID.randomUUID().toString()); Mockito.when(taskExecutionContext.getTenantCode()).thenReturn("root"); Mockito.when(taskExecutionContext.getStartTime()).thenReturn(new Date()); Mockito.when(taskExecutionContext.getTaskTimeout()).thenReturn(10000); @@ -110,28 +124,20 @@ public class DataxTaskTest { dataxTask = PowerMockito.spy(new DataxTask(taskExecutionContext, logger)); dataxTask.init(); props.setCmdTypeIfComplement(START_PROCESS); - setTaskParems(0); + Mockito.when(processService.findDataSourceById(1)).thenReturn(getDataSource()); Mockito.when(processService.findDataSourceById(2)).thenReturn(getDataSource()); Mockito.when(processService.findProcessInstanceByTaskId(1)).thenReturn(getProcessInstance()); String fileName = String.format("%s/%s_node.sh", props.getExecutePath(), props.getTaskAppId()); - Mockito.when(shellCommandExecutor.run(fileName)).thenReturn(null); - } - - private void setTaskParems(Integer customConfig) { - if (customConfig == 1) { - props.setTaskParams( - "{\"customConfig\":1, \"localParams\":[{\"prop\":\"test\",\"value\":\"38294729\"}],\"json\":\"{\\\"job\\\":{\\\"setting\\\":{\\\"speed\\\":{\\\"byte\\\":1048576},\\\"errorLimit\\\":{\\\"record\\\":0,\\\"percentage\\\":0.02}},\\\"content\\\":[{\\\"reader\\\":{\\\"name\\\":\\\"rdbmsreader\\\",\\\"parameter\\\":{\\\"username\\\":\\\"xxx\\\",\\\"password\\\":\\\"${test}\\\",\\\"column\\\":[\\\"id\\\",\\\"name\\\"],\\\"splitPk\\\":\\\"pk\\\",\\\"connection\\\":[{\\\"querySql\\\":[\\\"SELECT * from dual\\\"],\\\"jdbcUrl\\\":[\\\"jdbc:dm://ip:port/database\\\"]}],\\\"fetchSize\\\":1024,\\\"where\\\":\\\"1 = 1\\\"}},\\\"writer\\\":{\\\"name\\\":\\\"streamwriter\\\",\\\"parameter\\\":{\\\"print\\\":true}}}]}}\"}"); - -// "{\"customConfig\":1,\"json\":\"{\\\"job\\\":{\\\"setting\\\":{\\\"speed\\\":{\\\"byte\\\":1048576},\\\"errorLimit\\\":{\\\"record\\\":0,\\\"percentage\\\":0.02}},\\\"content\\\":[{\\\"reader\\\":{\\\"name\\\":\\\"rdbmsreader\\\",\\\"parameter\\\":{\\\"username\\\":\\\"xxx\\\",\\\"password\\\":\\\"xxx\\\",\\\"column\\\":[\\\"id\\\",\\\"name\\\"],\\\"splitPk\\\":\\\"pk\\\",\\\"connection\\\":[{\\\"querySql\\\":[\\\"SELECT * from dual\\\"],\\\"jdbcUrl\\\":[\\\"jdbc:dm://ip:port/database\\\"]}],\\\"fetchSize\\\":1024,\\\"where\\\":\\\"1 = 1\\\"}},\\\"writer\\\":{\\\"name\\\":\\\"streamwriter\\\",\\\"parameter\\\":{\\\"print\\\":true}}}]}}\"}"); - } else { - props.setTaskParams( - "{\"customConfig\":0,\"targetTable\":\"test\",\"postStatements\":[],\"jobSpeedByte\":1024,\"jobSpeedRecord\":1000,\"dtType\":\"MYSQL\",\"datasource\":1,\"dsType\":\"MYSQL\",\"datatarget\":2,\"jobSpeedByte\":0,\"sql\":\"select 1 as test from dual\",\"preStatements\":[\"delete from test\"],\"postStatements\":[\"delete from test\"]}"); - + try { + Mockito.when(shellCommandExecutor.run(fileName)).thenReturn(null); + } catch (Exception e) { + e.printStackTrace(); } + dataxTask = PowerMockito.spy(new DataxTask(taskExecutionContext, logger)); dataxTask.init(); } @@ -153,7 +159,8 @@ public class DataxTaskTest { @After public void after() - throws Exception {} + throws Exception { + } /** * Method: DataxTask() @@ -209,7 +216,7 @@ public class DataxTaskTest { */ @Test public void testParsingSqlColumnNames() - throws Exception { + throws Exception { try { BaseDataSource dataSource = DataSourceFactory.getDatasource(getDataSource().getType(), getDataSource().getConnectionParams()); @@ -223,8 +230,7 @@ public class DataxTaskTest { Assert.assertTrue(columns.length == 2); Assert.assertEquals("[`a`, `table`]", Arrays.toString(columns)); - } - catch (Exception e) { + } catch (Exception e) { Assert.fail(e.getMessage()); } } @@ -234,7 +240,7 @@ public class DataxTaskTest { */ @Test public void testTryGrammaticalAnalysisSqlColumnNames() - throws Exception { + throws Exception { try { Method method = DataxTask.class.getDeclaredMethod("tryGrammaticalAnalysisSqlColumnNames", DbType.class, String.class); method.setAccessible(true); @@ -245,8 +251,7 @@ public class DataxTaskTest { Assert.assertTrue(columns.length == 2); Assert.assertEquals("[a, b]", Arrays.toString(columns)); - } - catch (Exception e) { + } catch (Exception e) { Assert.fail(e.getMessage()); } } @@ -257,7 +262,7 @@ public class DataxTaskTest { */ @Test public void testTryExecuteSqlResolveColumnNames() - throws Exception { + throws Exception { // TODO: Test goes here... } @@ -267,21 +272,33 @@ public class DataxTaskTest { @Test public void testBuildDataxJsonFile() throws Exception { + try { setTaskParems(1); - buildDataJson(); - setTaskParems(0); - buildDataJson(); + Method method = DataxTask.class.getDeclaredMethod("buildDataxJsonFile"); + method.setAccessible(true); + String filePath = (String) method.invoke(dataxTask, null); + Assert.assertNotNull(filePath); } catch (Exception e) { Assert.fail(e.getMessage()); } } - public void buildDataJson() throws Exception { - Method method = DataxTask.class.getDeclaredMethod("buildDataxJsonFile"); - method.setAccessible(true); - String filePath = (String) method.invoke(dataxTask, null); - Assert.assertNotNull(filePath); + /** + * Method: buildDataxJsonFile() + */ + @Test + public void testBuildDataxJsonFile0() + throws Exception { + try { + setTaskParems(0); + Method method = DataxTask.class.getDeclaredMethod("buildDataxJsonFile"); + method.setAccessible(true); + String filePath = (String) method.invoke(dataxTask, null); + Assert.assertNotNull(filePath); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } } /** @@ -289,27 +306,26 @@ public class DataxTaskTest { */ @Test public void testBuildDataxJobContentJson() - throws Exception { + throws Exception { try { Method method = DataxTask.class.getDeclaredMethod("buildDataxJobContentJson"); method.setAccessible(true); - List contentList = (List) method.invoke(dataxTask, null); + List contentList = (List) method.invoke(dataxTask, null); Assert.assertNotNull(contentList); - JSONObject content = contentList.get(0); - JSONObject reader = (JSONObject) content.get("reader"); + ObjectNode content = contentList.get(0); + JsonNode reader = JSONUtils.parseObject(content.path("reader").asText()); Assert.assertNotNull(reader); - String readerPluginName = (String) reader.get("name"); + String readerPluginName = reader.path("name").asText(); Assert.assertEquals(DataxUtils.DATAX_READER_PLUGIN_MYSQL, readerPluginName); - JSONObject writer = (JSONObject) content.get("writer"); + JsonNode writer = JSONUtils.parseObject(content.path("writer").asText()); Assert.assertNotNull(writer); - String writerPluginName = (String) writer.get("name"); + String writerPluginName = writer.path("name").asText(); Assert.assertEquals(DataxUtils.DATAX_WRITER_PLUGIN_MYSQL, writerPluginName); - } - catch (Exception e) { + } catch (Exception e) { Assert.fail(e.getMessage()); } } @@ -319,16 +335,15 @@ public class DataxTaskTest { */ @Test public void testBuildDataxJobSettingJson() - throws Exception { + throws Exception { try { Method method = DataxTask.class.getDeclaredMethod("buildDataxJobSettingJson"); method.setAccessible(true); - JSONObject setting = (JSONObject) method.invoke(dataxTask, null); + JsonNode setting = (JsonNode) method.invoke(dataxTask, null); Assert.assertNotNull(setting); Assert.assertNotNull(setting.get("speed")); Assert.assertNotNull(setting.get("errorLimit")); - } - catch (Exception e) { + } catch (Exception e) { Assert.fail(e.getMessage()); } } @@ -338,15 +353,14 @@ public class DataxTaskTest { */ @Test public void testBuildDataxCoreJson() - throws Exception { + throws Exception { try { Method method = DataxTask.class.getDeclaredMethod("buildDataxCoreJson"); method.setAccessible(true); - JSONObject coreConfig = (JSONObject) method.invoke(dataxTask, null); + ObjectNode coreConfig = (ObjectNode) method.invoke(dataxTask, null); Assert.assertNotNull(coreConfig); Assert.assertNotNull(coreConfig.get("transport")); - } - catch (Exception e) { + } catch (Exception e) { Assert.fail(e.getMessage()); } } @@ -356,13 +370,12 @@ public class DataxTaskTest { */ @Test public void testBuildShellCommandFile() - throws Exception { + throws Exception { try { Method method = DataxTask.class.getDeclaredMethod("buildShellCommandFile", String.class); method.setAccessible(true); Assert.assertNotNull(method.invoke(dataxTask, "test.json")); - } - catch (Exception e) { + } catch (Exception e) { Assert.fail(e.getMessage()); } } @@ -381,13 +394,12 @@ public class DataxTaskTest { */ @Test public void testNotNull() - throws Exception { + throws Exception { try { Method method = DataxTask.class.getDeclaredMethod("notNull", Object.class, String.class); method.setAccessible(true); method.invoke(dataxTask, "abc", "test throw RuntimeException"); - } - catch (Exception e) { + } catch (Exception e) { Assert.fail(e.getMessage()); } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTaskTest.java index e50a54a372..f0d5d79d00 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTaskTest.java @@ -24,6 +24,7 @@ import java.util.Date; import org.apache.dolphinscheduler.common.enums.HttpCheckCondition; import org.apache.dolphinscheduler.common.enums.HttpMethod; import org.apache.dolphinscheduler.common.task.http.HttpParameters; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; @@ -47,7 +48,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; -import com.alibaba.fastjson.JSON; @RunWith(PowerMockRunner.class) @PrepareForTest(OSUtils.class) @@ -123,7 +123,7 @@ public class HttpTaskTest { public void testGenerator(){ String paramJson = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://github.com/\"," + "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}"; - HttpParameters httpParameters = JSON.parseObject(paramJson, HttpParameters.class); + HttpParameters httpParameters = JSONUtils.parseObject(paramJson, HttpParameters.class); Assert.assertEquals(10000,httpParameters.getConnectTimeout() ); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTaskTest.java index f0bcd9ec27..521650d3b6 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTaskTest.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.worker.task.spark; import org.apache.dolphinscheduler.common.enums.SparkVersion; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.spark.SparkParameters; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.server.utils.ParamUtils; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java index 0f18b4e2ac..2d0e39aa69 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java @@ -16,8 +16,8 @@ */ package org.apache.dolphinscheduler.server.worker.task.sqoop; -import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.server.entity.SqoopTaskExecutionContext; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.SqoopJobGenerator; @@ -86,7 +86,7 @@ public class SqoopTaskTest { //import mysql to HDFS with hadoo String mysqlToHdfs = "{\"jobName\":\"sqoop_import\",\"hadoopCustomParams\":[{\"prop\":\"mapreduce.map.memory.mb\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"4096\"}],\"sqoopAdvancedParams\":[{\"prop\":\"--direct\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"\"}]," + "\"jobType\":\"TEMPLATE\",\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HDFS\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"0\\\",\\\"srcQuerySql\\\":\\\"\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[]}\",\"targetParams\":\"{\\\"targetPath\\\":\\\"/ods/tmp/test/person7\\\",\\\"deleteTargetDir\\\":true,\\\"fileType\\\":\\\"--as-textfile\\\",\\\"compressionCodec\\\":\\\"\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters mysqlToHdfsParams = JSON.parseObject(mysqlToHdfs,SqoopParameters.class); + SqoopParameters mysqlToHdfsParams = JSONUtils.parseObject(mysqlToHdfs,SqoopParameters.class); SqoopJobGenerator generator = new SqoopJobGenerator(); String mysqlToHdfsScript = generator.generateSqoopJob(mysqlToHdfsParams,mysqlTaskExecutionContext); String mysqlToHdfsExpected = "sqoop import -D mapred.job.name=sqoop_import -D mapreduce.map.memory.mb=4096 --direct -m 1 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_2 --target-dir /ods/tmp/test/person7 --as-textfile --delete-target-dir --fields-terminated-by '@' --lines-terminated-by '\\n' --null-non-string 'NULL' --null-string 'NULL'"; @@ -96,28 +96,28 @@ public class SqoopTaskTest { String hdfsToMysql = "{\"jobName\":\"sqoop_import\",\"jobType\":\"TEMPLATE\",\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HDFS\"," + "\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"exportDir\\\":\\\"/ods/tmp/test/person7\\\"}\"," + "\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"id,name,age,sex,create_time\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":true,\\\"targetUpdateKey\\\":\\\"id\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters hdfsToMysqlParams = JSON.parseObject(hdfsToMysql,SqoopParameters.class); + SqoopParameters hdfsToMysqlParams = JSONUtils.parseObject(hdfsToMysql,SqoopParameters.class); String hdfsToMysqlScript = generator.generateSqoopJob(hdfsToMysqlParams,mysqlTaskExecutionContext); String hdfsToMysqlScriptExpected = "sqoop export -D mapred.job.name=sqoop_import -m 1 --export-dir /ods/tmp/test/person7 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --columns id,name,age,sex,create_time --fields-terminated-by '@' --lines-terminated-by '\\n' --update-key id --update-mode allowinsert"; Assert.assertEquals(hdfsToMysqlScriptExpected, hdfsToMysqlScript); //export hive to mysql String hiveToMysql = "{\"jobName\":\"sqoop_import\",\"jobType\":\"TEMPLATE\",\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HIVE\",\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-17\\\"}\",\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":false,\\\"targetUpdateKey\\\":\\\"\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters hiveToMysqlParams = JSON.parseObject(hiveToMysql,SqoopParameters.class); + SqoopParameters hiveToMysqlParams = JSONUtils.parseObject(hiveToMysql,SqoopParameters.class); String hiveToMysqlScript = generator.generateSqoopJob(hiveToMysqlParams,mysqlTaskExecutionContext); String hiveToMysqlExpected = "sqoop export -D mapred.job.name=sqoop_import -m 1 --hcatalog-database stg --hcatalog-table person_internal --hcatalog-partition-keys date --hcatalog-partition-values 2020-02-17 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --fields-terminated-by '@' --lines-terminated-by '\\n'"; Assert.assertEquals(hiveToMysqlExpected, hiveToMysqlScript); //import mysql to hive String mysqlToHive = "{\"jobName\":\"sqoop_import\",\"jobType\":\"TEMPLATE\",\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HIVE\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"1\\\",\\\"srcQuerySql\\\":\\\"SELECT * FROM person_2\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[{\\\"prop\\\":\\\"id\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"Integer\\\"}]}\",\"targetParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal_2\\\",\\\"createHiveTable\\\":true,\\\"dropDelimiter\\\":false,\\\"hiveOverWrite\\\":true,\\\"replaceDelimiter\\\":\\\"\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-16\\\"}\",\"localParams\":[]}"; - SqoopParameters mysqlToHiveParams = JSON.parseObject(mysqlToHive,SqoopParameters.class); + SqoopParameters mysqlToHiveParams = JSONUtils.parseObject(mysqlToHive,SqoopParameters.class); String mysqlToHiveScript = generator.generateSqoopJob(mysqlToHiveParams,mysqlTaskExecutionContext); String mysqlToHiveExpected = "sqoop import -D mapred.job.name=sqoop_import -m 1 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --query 'SELECT * FROM person_2 WHERE $CONDITIONS' --map-column-java id=Integer --hive-import --hive-table stg.person_internal_2 --create-hive-table --hive-overwrite -delete-target-dir --hive-partition-key date --hive-partition-value 2020-02-16"; Assert.assertEquals(mysqlToHiveExpected, mysqlToHiveScript); //sqoop CUSTOM job String sqoopCustomString = "{\"jobType\":\"CUSTOM\",\"localParams\":[],\"customShell\":\"sqoop import\"}"; - SqoopParameters sqoopCustomParams = JSON.parseObject(sqoopCustomString, SqoopParameters.class); + SqoopParameters sqoopCustomParams = JSONUtils.parseObject(sqoopCustomString, SqoopParameters.class); String sqoopCustomScript = generator.generateSqoopJob(sqoopCustomParams, new TaskExecutionContext()); String sqoopCustomExpected = "sqoop import"; Assert.assertEquals(sqoopCustomExpected, sqoopCustomScript); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java index 8e63c89405..c4fe4475cc 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java @@ -21,7 +21,7 @@ import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.log.*; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; +import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,7 +77,7 @@ public class LogClientService { Command command = request.convert2Command(); Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); if(response != null){ - RollViewLogResponseCommand rollReviewLog = FastJsonSerializer.deserialize( + RollViewLogResponseCommand rollReviewLog = JsonSerializer.deserialize( response.getBody(), RollViewLogResponseCommand.class); return rollReviewLog.getMsg(); } @@ -105,7 +105,7 @@ public class LogClientService { Command command = request.convert2Command(); Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); if(response != null){ - ViewLogResponseCommand viewLog = FastJsonSerializer.deserialize( + ViewLogResponseCommand viewLog = JsonSerializer.deserialize( response.getBody(), ViewLogResponseCommand.class); return viewLog.getMsg(); } @@ -133,7 +133,7 @@ public class LogClientService { Command command = request.convert2Command(); Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); if(response != null){ - GetLogBytesResponseCommand getLog = FastJsonSerializer.deserialize( + GetLogBytesResponseCommand getLog = JsonSerializer.deserialize( response.getBody(), GetLogBytesResponseCommand.class); return getLog.getData(); } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 73f7defe17..60c17f7f18 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -16,9 +16,8 @@ */ package org.apache.dolphinscheduler.service.process; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; import com.cronutils.model.Cron; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.lang.ArrayUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.*; @@ -205,16 +204,15 @@ public class ProcessService { CommandType commandType = command.getCommandType(); if(cmdTypeMap.containsKey(commandType)){ - JSONObject cmdParamObj = (JSONObject) JSON.parse(command.getCommandParam()); - JSONObject tempObj; - int processInstanceId = cmdParamObj.getInteger(CMDPARAM_RECOVER_PROCESS_ID_STRING); + ObjectNode cmdParamObj = JSONUtils.parseObject(command.getCommandParam()); + int processInstanceId = cmdParamObj.path(CMDPARAM_RECOVER_PROCESS_ID_STRING).asInt(); List commands = commandMapper.selectList(null); // for all commands for (Command tmpCommand:commands){ if(cmdTypeMap.containsKey(tmpCommand.getCommandType())){ - tempObj = (JSONObject) JSON.parse(tmpCommand.getCommandParam()); - if(tempObj != null && processInstanceId == tempObj.getInteger(CMDPARAM_RECOVER_PROCESS_ID_STRING)){ + ObjectNode tempObj = JSONUtils.parseObject(tmpCommand.getCommandParam()); + if(tempObj != null && processInstanceId == tempObj.path(CMDPARAM_RECOVER_PROCESS_ID_STRING).asInt()){ isNeedCreate = false; break; } @@ -331,7 +329,7 @@ public class ProcessService { for (TaskNode taskNode : taskNodeList){ String parameter = taskNode.getParams(); if (parameter.contains(CMDPARAM_SUB_PROCESS_DEFINE_ID)){ - SubProcessParameters subProcessParam = JSON.parseObject(parameter, SubProcessParameters.class); + SubProcessParameters subProcessParam = JSONUtils.parseObject(parameter, SubProcessParameters.class); ids.add(subProcessParam.getProcessDefinitionId()); recurseFindSubProcessId(subProcessParam.getProcessDefinitionId(),ids); } @@ -366,7 +364,7 @@ public class ProcessService { processInstance.getFailureStrategy(), processInstance.getExecutorId(), processInstance.getProcessDefinitionId(), - JSONUtils.toJson(cmdParam), + JSONUtils.toJsonString(cmdParam), processInstance.getWarningType(), processInstance.getWarningGroupId(), processInstance.getScheduleTime(), @@ -386,7 +384,7 @@ public class ProcessService { originCommand.setId(0); originCommand.setCommandType(CommandType.RECOVER_WAITTING_THREAD); originCommand.setUpdateTime(new Date()); - originCommand.setCommandParam(JSONUtils.toJson(cmdParam)); + originCommand.setCommandParam(JSONUtils.toJsonString(cmdParam)); originCommand.setProcessInstancePriority(processInstance.getProcessInstancePriority()); saveCommand(originCommand); } @@ -600,7 +598,7 @@ public class ProcessService { } cmdParam.put(Constants.CMDPARAM_RECOVERY_START_NODE_STRING, String.join(Constants.COMMA, convertIntListToString(failedList))); - processInstance.setCommandParam(JSONUtils.toJson(cmdParam)); + processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam)); processInstance.setRunTimes(runTime +1 ); break; case START_CURRENT_TASK_PROCESS: @@ -619,7 +617,7 @@ public class ProcessService { initTaskInstance(this.findTaskInstanceById(taskId)); } cmdParam.put(Constants.CMDPARAM_RECOVERY_START_NODE_STRING, String.join(",", convertIntListToString(suspendedNodeList))); - processInstance.setCommandParam(JSONUtils.toJson(cmdParam)); + processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam)); processInstance.setRunTimes(runTime +1); break; case RECOVER_TOLERANCE_FAULT_PROCESS: @@ -639,7 +637,7 @@ public class ProcessService { // delete the recover task names from command parameter if(cmdParam.containsKey(Constants.CMDPARAM_RECOVERY_START_NODE_STRING)){ cmdParam.remove(Constants.CMDPARAM_RECOVERY_START_NODE_STRING); - processInstance.setCommandParam(JSONUtils.toJson(cmdParam)); + processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam)); } // delete all the valid tasks when repeat running List validTaskList = findValidTaskListByProcessId(processInstance.getId()); @@ -717,7 +715,7 @@ public class ProcessService { && CMDPARAM_EMPTY_SUB_PROCESS.equals(paramMap.get(CMDPARAM_SUB_PROCESS))){ paramMap.remove(CMDPARAM_SUB_PROCESS); paramMap.put(CMDPARAM_SUB_PROCESS, String.valueOf(subProcessInstance.getId())); - subProcessInstance.setCommandParam(JSONUtils.toJson(paramMap)); + subProcessInstance.setCommandParam(JSONUtils.toJsonString(paramMap)); subProcessInstance.setIsSubProcess(Flag.YES); this.saveProcessInstance(subProcessInstance); } @@ -752,8 +750,10 @@ public class ProcessService { * @return global params join */ private String joinGlobalParams(String parentGlobalParams, String subGlobalParams){ + List parentPropertyList = JSONUtils.toList(parentGlobalParams, Property.class); List subPropertyList = JSONUtils.toList(subGlobalParams, Property.class); + Map subMap = subPropertyList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); for(Property parent : parentPropertyList){ @@ -761,7 +761,7 @@ public class ProcessService { subPropertyList.add(parent); } } - return JSONUtils.toJson(subPropertyList); + return JSONUtils.toJsonString(subPropertyList); } /** @@ -896,7 +896,7 @@ public class ProcessService { updateProcessInstance(childInstance); } // set sub work process command - String processMapStr = JSONUtils.toJson(instanceMap); + String processMapStr = JSONUtils.toJsonString(instanceMap); Map cmdParam = JSONUtils.toMap(processMapStr); if(commandType == CommandType.COMPLEMENT_DATA || @@ -906,7 +906,7 @@ public class ProcessService { String startTime = parentParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, endTime); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, startTime); - processMapStr = JSONUtils.toJson(cmdParam); + processMapStr = JSONUtils.toJsonString(cmdParam); } updateSubProcessDefinitionByParent(parentProcessInstance, childDefineId); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java index 69ca97a3d8..01d7b2f1e5 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java @@ -20,7 +20,7 @@ import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.lang.StringUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.Schedule; import org.quartz.*; import org.quartz.impl.StdSchedulerFactory; @@ -334,7 +334,7 @@ public class QuartzExecutors { Map dataMap = new HashMap<>(3); dataMap.put(PROJECT_ID, projectId); dataMap.put(SCHEDULE_ID, scheduleId); - dataMap.put(SCHEDULE, JSONUtils.toJson(schedule)); + dataMap.put(SCHEDULE, JSONUtils.toJsonString(schedule)); return dataMap; } diff --git a/pom.xml b/pom.xml index d532bccc1f..bdac1f16aa 100644 --- a/pom.xml +++ b/pom.xml @@ -69,7 +69,6 @@ 3.2.0 2.0.1 5.0.5 - 1.2.61 1.1.14 1.4.200 1.6 @@ -150,11 +149,6 @@ ${cron.utils.version} - - com.alibaba - fastjson - ${fastjson.version} - com.alibaba druid @@ -787,7 +781,7 @@ **/dao/mapper/CommandMapperTest.java **/dao/mapper/ConnectionFactoryTest.java **/dao/mapper/DataSourceMapperTest.java - **/remote/FastJsonSerializerTest.java + **/remote/JsonSerializerTest.java **/remote/NettyRemotingClientTest.java **/remote/ResponseFutureTest.java **/server/log/MasterLogFilterTest.java @@ -822,6 +816,7 @@ **/server/worker/task/spark/SparkTaskTest.java **/server/worker/task/EnvFileTest.java **/server/worker/task/spark/SparkTaskTest.java + **/server/worker/task/datax/DataxTaskTest.java **/server/worker/task/http/HttpTaskTest.java **/server/worker/task/sqoop/SqoopTaskTest.java **/server/worker/EnvFileTest.java diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index b1c7f5f806..f045bb275c 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -43,7 +43,6 @@ datanucleus-core-4.1.6.jar datanucleus-rdbms-4.1.7.jar derby-10.14.2.0.jar druid-1.1.14.jar -fastjson-1.2.61.jar gson-2.8.5.jar guava-20.0.jar guice-3.0.jar