github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

use static method in static class JSON (#2156)

pull/2/head
gabry.wu 5 years ago committed by GitHub
parent
dd2c50a10d
commit
daad5efbab
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 79 additions and 74 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 6
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java
  2. 25
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java
  3. 23
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
  4. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java
  5. 15
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java
  6. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
  7. 4
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java
  8. 8
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java
  9. 8
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java
  10. 4
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java
  11. 4
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java
  12. 6
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java
  13. 4
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java
  14. 9
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
  15. 4
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java
  16. 4
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java
  17. 4
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java
  18. 10
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java
  19. 7
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java

6
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java
Unescape View File

@ -16,8 +16,8 @@
*/ */
package org.apache.dolphinscheduler.alert.utils; package org.apache.dolphinscheduler.alert.utils;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -38,7 +38,7 @@ public class JSONUtils {
*/ */
public static String toJsonString(Object object) { public static String toJsonString(Object object) {
try{ try{
return JSONObject.toJSONString(object,false); return JSON.toJSONString(object,false);
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException("Json deserialization exception.", e); throw new RuntimeException("Json deserialization exception.", e);
} }
@ -57,7 +57,7 @@ public class JSONUtils {
return null; return null;
} }
try { try {
return JSONArray.parseArray(json, clazz); return JSON.parseArray(json, clazz);
} catch (Exception e) { } catch (Exception e) {
logger.error("JSONArray.parseArray exception!",e); logger.error("JSONArray.parseArray exception!",e);
} }

25
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java
Unescape View File

@ -16,6 +16,7 @@
*/ */
package org.apache.dolphinscheduler.api.service; package org.apache.dolphinscheduler.api.service;
import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
@ -303,7 +304,7 @@ public class DataSourceService extends BaseService{
for (DataSource dataSource : dataSourceList) { for (DataSource dataSource : dataSourceList) {
String connectionParams = dataSource.getConnectionParams(); String connectionParams = dataSource.getConnectionParams();
JSONObject object = JSONObject.parseObject(connectionParams); JSONObject object = JSON.parseObject(connectionParams);
object.put(Constants.PASSWORD, Constants.XXXXXX); object.put(Constants.PASSWORD, Constants.XXXXXX);
dataSource.setConnectionParams(JSONUtils.toJson(object)); dataSource.setConnectionParams(JSONUtils.toJson(object));
@ -367,11 +368,11 @@ public class DataSourceService extends BaseService{
try { try {
switch (dbType) { switch (dbType) {
case POSTGRESQL: case POSTGRESQL:
datasource = JSONObject.parseObject(parameter, PostgreDataSource.class); datasource = JSON.parseObject(parameter, PostgreDataSource.class);
Class.forName(Constants.ORG_POSTGRESQL_DRIVER); Class.forName(Constants.ORG_POSTGRESQL_DRIVER);
break; break;
case MYSQL: case MYSQL:
datasource = JSONObject.parseObject(parameter, MySQLDataSource.class); datasource = JSON.parseObject(parameter, MySQLDataSource.class);
Class.forName(Constants.COM_MYSQL_JDBC_DRIVER); Class.forName(Constants.COM_MYSQL_JDBC_DRIVER);
break; break;
case HIVE: case HIVE:
@ -386,26 +387,26 @@ public class DataSourceService extends BaseService{
getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_PATH)); getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_PATH));
} }
if (dbType == DbType.HIVE){ if (dbType == DbType.HIVE){
datasource = JSONObject.parseObject(parameter, HiveDataSource.class); datasource = JSON.parseObject(parameter, HiveDataSource.class);
}else if (dbType == DbType.SPARK){ }else if (dbType == DbType.SPARK){
datasource = JSONObject.parseObject(parameter, SparkDataSource.class); datasource = JSON.parseObject(parameter, SparkDataSource.class);
} }
Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER); Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER);
break; break;
case CLICKHOUSE: case CLICKHOUSE:
datasource = JSONObject.parseObject(parameter, ClickHouseDataSource.class); datasource = JSON.parseObject(parameter, ClickHouseDataSource.class);
Class.forName(Constants.COM_CLICKHOUSE_JDBC_DRIVER); Class.forName(Constants.COM_CLICKHOUSE_JDBC_DRIVER);
break; break;
case ORACLE: case ORACLE:
datasource = JSONObject.parseObject(parameter, OracleDataSource.class); datasource = JSON.parseObject(parameter, OracleDataSource.class);
Class.forName(Constants.COM_ORACLE_JDBC_DRIVER); Class.forName(Constants.COM_ORACLE_JDBC_DRIVER);
break; break;
case SQLSERVER: case SQLSERVER:
datasource = JSONObject.parseObject(parameter, SQLServerDataSource.class); datasource = JSON.parseObject(parameter, SQLServerDataSource.class);
Class.forName(Constants.COM_SQLSERVER_JDBC_DRIVER); Class.forName(Constants.COM_SQLSERVER_JDBC_DRIVER);
break; break;
case DB2: case DB2:
datasource = JSONObject.parseObject(parameter, DB2ServerDataSource.class); datasource = JSON.parseObject(parameter, DB2ServerDataSource.class);
Class.forName(Constants.COM_DB2_JDBC_DRIVER); Class.forName(Constants.COM_DB2_JDBC_DRIVER);
break; break;
default: default:
@ -507,7 +508,7 @@ public class DataSourceService extends BaseService{
parameterMap.put(Constants.PRINCIPAL,principal); parameterMap.put(Constants.PRINCIPAL,principal);
} }
if (other != null && !"".equals(other)) { if (other != null && !"".equals(other)) {
LinkedHashMap<String, String> map = JSONObject.parseObject(other, new TypeReference<LinkedHashMap<String, String>>() { LinkedHashMap<String, String> map = JSON.parseObject(other, new TypeReference<LinkedHashMap<String, String>>() {
}); });
if (map.size() > 0) { if (map.size() > 0) {
StringBuilder otherSb = new StringBuilder(); StringBuilder otherSb = new StringBuilder();
@ -523,9 +524,9 @@ public class DataSourceService extends BaseService{
} }
if(logger.isDebugEnabled()){ if(logger.isDebugEnabled()){
logger.info("parameters map-----" + JSONObject.toJSONString(parameterMap)); logger.info("parameters map-----" + JSON.toJSONString(parameterMap));
} }
return JSONObject.toJSONString(parameterMap); return JSON.toJSONString(parameterMap);
} }

23
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
Unescape View File

@ -16,6 +16,7 @@
*/ */
package org.apache.dolphinscheduler.api.controller; package org.apache.dolphinscheduler.api.controller;
import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.ResourceType;
@ -54,7 +55,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode()); result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -78,7 +79,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode()); result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -281,7 +282,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode()); result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -303,7 +304,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode()); result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -324,7 +325,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode()); result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -344,7 +345,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode()); result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -365,7 +366,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode()); result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -386,7 +387,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode()); result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -406,7 +407,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode()); result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -427,7 +428,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode()); result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -446,7 +447,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode()); result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java
Unescape View File

@ -16,6 +16,7 @@
*/ */
package org.apache.dolphinscheduler.common.model; package org.apache.dolphinscheduler.common.model;
import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.Priority;
import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy;
@ -23,7 +24,6 @@ import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize;
@ -294,7 +294,7 @@ public class TaskNode {
if(StringUtils.isNotEmpty(this.getTimeout())){ if(StringUtils.isNotEmpty(this.getTimeout())){
String formatStr = String.format("%s,%s", TaskTimeoutStrategy.WARN.name(), TaskTimeoutStrategy.FAILED.name()); String formatStr = String.format("%s,%s", TaskTimeoutStrategy.WARN.name(), TaskTimeoutStrategy.FAILED.name());
String timeout = this.getTimeout().replace(formatStr,TaskTimeoutStrategy.WARNFAILED.name()); String timeout = this.getTimeout().replace(formatStr,TaskTimeoutStrategy.WARNFAILED.name());
return JSONObject.parseObject(timeout,TaskTimeoutParameter.class); return JSON.parseObject(timeout,TaskTimeoutParameter.class);
} }
return new TaskTimeoutParameter(false); return new TaskTimeoutParameter(false);
} }

15
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java
Unescape View File

@ -16,6 +16,7 @@
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.TypeReference; import com.alibaba.fastjson.TypeReference;
@ -59,7 +60,7 @@ public class JSONUtils {
*/ */
public static String toJson(Object object) { public static String toJson(Object object) {
try{ try{
return JSONObject.toJSONString(object,false); return JSON.toJSONString(object,false);
} catch (Exception e) { } catch (Exception e) {
logger.error("object to json exception!",e); logger.error("object to json exception!",e);
} }
@ -89,7 +90,7 @@ public class JSONUtils {
} }
try { try {
return JSONObject.parseObject(json, clazz); return JSON.parseObject(json, clazz);
} catch (Exception e) { } catch (Exception e) {
logger.error("parse object exception!",e); logger.error("parse object exception!",e);
} }
@ -178,7 +179,7 @@ public class JSONUtils {
} }
try { try {
return JSONObject.parseObject(json, new TypeReference<HashMap<String, String>>(){}); return JSON.parseObject(json, new TypeReference<HashMap<String, String>>(){});
} catch (Exception e) { } catch (Exception e) {
logger.error("json to map exception!",e); logger.error("json to map exception!",e);
} }
@ -203,7 +204,7 @@ public class JSONUtils {
} }
try { try {
return JSONObject.parseObject(json, new TypeReference<HashMap<K, V>>() {}); return JSON.parseObject(json, new TypeReference<HashMap<K, V>>() {});
} catch (Exception e) { } catch (Exception e) {
logger.error("json to map exception!",e); logger.error("json to map exception!",e);
} }
@ -218,7 +219,7 @@ public class JSONUtils {
*/ */
public static String toJsonString(Object object) { public static String toJsonString(Object object) {
try{ try{
return JSONObject.toJSONString(object,false); return JSON.toJSONString(object,false);
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException("Json deserialization exception.", e); throw new RuntimeException("Json deserialization exception.", e);
} }
@ -226,7 +227,7 @@ public class JSONUtils {
public static JSONObject parseObject(String text) { public static JSONObject parseObject(String text) {
try{ try{
return JSONObject.parseObject(text); return JSON.parseObject(text);
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException("Json deserialization exception.", e); throw new RuntimeException("Json deserialization exception.", e);
} }
@ -234,7 +235,7 @@ public class JSONUtils {
public static JSONArray parseArray(String text) { public static JSONArray parseArray(String text) {
try{ try{
return JSONObject.parseArray(text); return JSON.parseArray(text);
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException("Json deserialization exception.", e); throw new RuntimeException("Json deserialization exception.", e);
} }

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
Unescape View File

@ -16,6 +16,7 @@
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.DataType;
@ -23,7 +24,6 @@ import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils; import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils; import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.time.DateUtils; import org.apache.commons.lang.time.DateUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -157,7 +157,7 @@ public class ParameterUtils {
property.setValue(val); property.setValue(val);
} }
} }
return JSONObject.toJSONString(globalParamList); return JSON.toJSONString(globalParamList);
} }

4
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java
Unescape View File

@ -16,10 +16,10 @@
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.DataType;
import org.apache.dolphinscheduler.common.enums.Direct; import org.apache.dolphinscheduler.common.enums.Direct;
import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.process.Property;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import org.junit.Assert; import org.junit.Assert;
@ -53,7 +53,7 @@ public class JSONUtilsTest {
property.setType(DataType.VARCHAR); property.setType(DataType.VARCHAR);
property.setValue("sssssss"); property.setValue("sssssss");
String str = "{\"direct\":\"IN\",\"prop\":\"ds\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"}"; String str = "{\"direct\":\"IN\",\"prop\":\"ds\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"}";
Property property1 = JSONObject.parseObject(str, Property.class); Property property1 = JSON.parseObject(str, Property.class);
Direct direct = property1.getDirect(); Direct direct = property1.getDirect();
Assert.assertEquals(direct , Direct.IN); Assert.assertEquals(direct , Direct.IN);
} }

8
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java
Unescape View File

@ -16,7 +16,7 @@
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSON;
import org.apache.commons.lang.time.DateUtils; import org.apache.commons.lang.time.DateUtils;
import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.DataType;
@ -91,13 +91,13 @@ public class ParameterUtilsTest {
globalParamList.add(property); globalParamList.add(property);
String result2 = ParameterUtils.curingGlobalParams(null,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,scheduleTime); String result2 = ParameterUtils.curingGlobalParams(null,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,scheduleTime);
Assert.assertEquals(result2, JSONObject.toJSONString(globalParamList)); Assert.assertEquals(result2, JSON.toJSONString(globalParamList));
String result3 = ParameterUtils.curingGlobalParams(globalParamMap,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,null); String result3 = ParameterUtils.curingGlobalParams(globalParamMap,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,null);
Assert.assertEquals(result3, JSONObject.toJSONString(globalParamList)); Assert.assertEquals(result3, JSON.toJSONString(globalParamList));
String result4 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime); String result4 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime);
Assert.assertEquals(result4, JSONObject.toJSONString(globalParamList)); Assert.assertEquals(result4, JSON.toJSONString(globalParamList));
//test var $ startsWith //test var $ startsWith
globalParamMap.put("bizDate","${system.biz.date}"); globalParamMap.put("bizDate","${system.biz.date}");

8
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java
Unescape View File

@ -16,10 +16,10 @@
*/ */
package org.apache.dolphinscheduler.dao.entity; package org.apache.dolphinscheduler.dao.entity;
import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.ReleaseState;
import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.process.Property;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableId;
@ -266,7 +266,7 @@ public class ProcessDefinition {
} }
public void setGlobalParams(String globalParams) { public void setGlobalParams(String globalParams) {
this.globalParamList = JSONObject.parseArray(globalParams, Property.class); this.globalParamList = JSON.parseArray(globalParams, Property.class);
this.globalParams = globalParams; this.globalParams = globalParams;
} }
@ -275,7 +275,7 @@ public class ProcessDefinition {
} }
public void setGlobalParamList(List<Property> globalParamList) { public void setGlobalParamList(List<Property> globalParamList) {
this.globalParams = JSONObject.toJSONString(globalParamList); this.globalParams = JSON.toJSONString(globalParamList);
this.globalParamList = globalParamList; this.globalParamList = globalParamList;
} }
@ -283,7 +283,7 @@ public class ProcessDefinition {
List<Property> propList; List<Property> propList;
if (globalParamMap == null && StringUtils.isNotEmpty(globalParams)) { if (globalParamMap == null && StringUtils.isNotEmpty(globalParams)) {
propList = JSONObject.parseArray(globalParams, Property.class); propList = JSON.parseArray(globalParams, Property.class);
globalParamMap = propList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); globalParamMap = propList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue));
} }

4
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java
Unescape View File

@ -16,7 +16,7 @@
*/ */
package org.apache.dolphinscheduler.server.master.runner; package org.apache.dolphinscheduler.server.master.runner;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSON;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
@ -454,7 +454,7 @@ public class MasterExecThread implements Runnable {
// process instance id // process instance id
taskInstance.setProcessInstanceId(processInstance.getId()); taskInstance.setProcessInstanceId(processInstance.getId());
// task instance node json // task instance node json
taskInstance.setTaskJson(JSONObject.toJSONString(taskNode)); taskInstance.setTaskJson(JSON.toJSONString(taskNode));
// task instance type // task instance type
taskInstance.setTaskType(taskNode.getType()); taskInstance.setTaskType(taskNode.getType());
// task instance whether alert // task instance whether alert

4
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java
Unescape View File

@ -16,6 +16,7 @@
*/ */
package org.apache.dolphinscheduler.server.master.runner; package org.apache.dolphinscheduler.server.master.runner;
import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy;
@ -25,7 +26,6 @@ import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import com.alibaba.fastjson.JSONObject;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -176,7 +176,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread {
*/ */
private TaskTimeoutParameter getTaskTimeoutParameter(){ private TaskTimeoutParameter getTaskTimeoutParameter(){
String taskJson = taskInstance.getTaskJson(); String taskJson = taskInstance.getTaskJson();
TaskNode taskNode = JSONObject.parseObject(taskJson, TaskNode.class); TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class);
return taskNode.getTaskTimeoutParameter(); return taskNode.getTaskTimeoutParameter();
} }

6
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java
Unescape View File

@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.worker.runner;
import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.sift.SiftingAppender; import ch.qos.logback.classic.sift.SiftingAppender;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.AuthorizationType;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
@ -93,7 +93,7 @@ public class TaskScheduleThread implements Runnable {
logger.info("script path : {}", taskInstance.getExecutePath()); logger.info("script path : {}", taskInstance.getExecutePath());
// task node // task node
TaskNode taskNode = JSONObject.parseObject(taskInstance.getTaskJson(), TaskNode.class); TaskNode taskNode = JSON.parseObject(taskInstance.getTaskJson(), TaskNode.class);
// get resource files // get resource files
List<String> resourceFiles = createProjectResFiles(taskNode); List<String> resourceFiles = createProjectResFiles(taskNode);
@ -176,7 +176,7 @@ public class TaskScheduleThread implements Runnable {
String globalParamsStr = taskInstance.getProcessInstance().getGlobalParams(); String globalParamsStr = taskInstance.getProcessInstance().getGlobalParams();
if (globalParamsStr != null) { if (globalParamsStr != null) {
List<Property> globalParamsList = JSONObject.parseArray(globalParamsStr, Property.class); List<Property> globalParamsList = JSON.parseArray(globalParamsStr, Property.class);
globalParamsMap.putAll(globalParamsList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue))); globalParamsMap.putAll(globalParamsList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)));
} }
return globalParamsMap; return globalParamsMap;

4
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java
Unescape View File

@ -16,7 +16,7 @@
*/ */
package org.apache.dolphinscheduler.server.worker.task.processdure; package org.apache.dolphinscheduler.server.worker.task.processdure;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSON;
import com.cronutils.utils.StringUtils; import com.cronutils.utils.StringUtils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.DataType;
@ -75,7 +75,7 @@ public class ProcedureTask extends AbstractTask {
logger.info("procedure task params {}", taskProps.getTaskParams()); logger.info("procedure task params {}", taskProps.getTaskParams());
this.procedureParameters = JSONObject.parseObject(taskProps.getTaskParams(), ProcedureParameters.class); this.procedureParameters = JSON.parseObject(taskProps.getTaskParams(), ProcedureParameters.class);
// check parameters // check parameters
if (!procedureParameters.checkParameters()) { if (!procedureParameters.checkParameters()) {

9
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
Unescape View File

@ -16,6 +16,7 @@
*/ */
package org.apache.dolphinscheduler.server.worker.task.sql; package org.apache.dolphinscheduler.server.worker.task.sql;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.serializer.SerializerFeature; import com.alibaba.fastjson.serializer.SerializerFeature;
@ -92,7 +93,7 @@ public class SqlTask extends AbstractTask {
super(taskProps, logger); super(taskProps, logger);
logger.info("sql task params {}", taskProps.getTaskParams()); logger.info("sql task params {}", taskProps.getTaskParams());
this.sqlParameters = JSONObject.parseObject(taskProps.getTaskParams(), SqlParameters.class); this.sqlParameters = JSON.parseObject(taskProps.getTaskParams(), SqlParameters.class);
if (!sqlParameters.checkParameters()) { if (!sqlParameters.checkParameters()) {
throw new RuntimeException("sql task params is not valid"); throw new RuntimeException("sql task params is not valid");
@ -308,16 +309,16 @@ public class SqlTask extends AbstractTask {
} }
resultJSONArray.add(mapOfColValues); resultJSONArray.add(mapOfColValues);
} }
logger.debug("execute sql : {}", JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); logger.debug("execute sql : {}", JSON.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue));
// if there is a result set // if there is a result set
if ( !resultJSONArray.isEmpty() ) { if ( !resultJSONArray.isEmpty() ) {
if (StringUtils.isNotEmpty(sqlParameters.getTitle())) { if (StringUtils.isNotEmpty(sqlParameters.getTitle())) {
sendAttachment(sqlParameters.getTitle(), sendAttachment(sqlParameters.getTitle(),
JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); JSON.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue));
}else{ }else{
sendAttachment(taskProps.getNodeName() + " query resultsets ", sendAttachment(taskProps.getNodeName() + " query resultsets ",
JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); JSON.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue));
} }
} }

4
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java
Unescape View File

@ -16,7 +16,7 @@
*/ */
package org.apache.dolphinscheduler.server.master; package org.apache.dolphinscheduler.server.master;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.common.enums.*; import org.apache.dolphinscheduler.common.enums.*;
import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.graph.DAG;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
@ -85,7 +85,7 @@ public class MasterExecThreadTest {
Map<String, String> cmdParam = new HashMap<>(); Map<String, String> cmdParam = new HashMap<>();
cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, "2020-01-01 00:00:00"); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, "2020-01-01 00:00:00");
cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, "2020-01-31 23:00:00"); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, "2020-01-31 23:00:00");
Mockito.when(processInstance.getCommandParam()).thenReturn(JSONObject.toJSONString(cmdParam)); Mockito.when(processInstance.getCommandParam()).thenReturn(JSON.toJSONString(cmdParam));
ProcessDefinition processDefinition = new ProcessDefinition(); ProcessDefinition processDefinition = new ProcessDefinition();
processDefinition.setGlobalParamMap(Collections.EMPTY_MAP); processDefinition.setGlobalParamMap(Collections.EMPTY_MAP);
processDefinition.setGlobalParamList(Collections.EMPTY_LIST); processDefinition.setGlobalParamList(Collections.EMPTY_LIST);

4
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java
Unescape View File

@ -16,7 +16,7 @@
*/ */
package org.apache.dolphinscheduler.server.worker.shell; package org.apache.dolphinscheduler.server.worker.shell;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNode;
@ -68,7 +68,7 @@ public class ShellCommandExecutorTest {
TaskInstance taskInstance = processService.findTaskInstanceById(7657); TaskInstance taskInstance = processService.findTaskInstanceById(7657);
String taskJson = taskInstance.getTaskJson(); String taskJson = taskInstance.getTaskJson();
TaskNode taskNode = JSONObject.parseObject(taskJson, TaskNode.class); TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class);
taskProps.setTaskParams(taskNode.getParams()); taskProps.setTaskParams(taskNode.getParams());

4
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java
Unescape View File

@ -16,7 +16,7 @@
*/ */
package org.apache.dolphinscheduler.server.worker.sql; package org.apache.dolphinscheduler.server.worker.sql;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
@ -112,7 +112,7 @@ public class SqlExecutorTest {
TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId);
String taskJson = taskInstance.getTaskJson(); String taskJson = taskInstance.getTaskJson();
TaskNode taskNode = JSONObject.parseObject(taskJson, TaskNode.class); TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class);
taskProps.setTaskParams(taskNode.getParams()); taskProps.setTaskParams(taskNode.getParams());

10
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java
Unescape View File

@ -16,7 +16,7 @@
*/ */
package org.apache.dolphinscheduler.server.worker.task.sqoop; package org.apache.dolphinscheduler.server.worker.task.sqoop;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.DataSource;
@ -74,7 +74,7 @@ public class SqoopTaskTest {
@Test @Test
public void testGenerator(){ public void testGenerator(){
String data1 = "{\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HDFS\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"0\\\",\\\"srcQuerySql\\\":\\\"\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[]}\",\"targetParams\":\"{\\\"targetPath\\\":\\\"/ods/tmp/test/person7\\\",\\\"deleteTargetDir\\\":true,\\\"fileType\\\":\\\"--as-textfile\\\",\\\"compressionCodec\\\":\\\"\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; String data1 = "{\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HDFS\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"0\\\",\\\"srcQuerySql\\\":\\\"\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[]}\",\"targetParams\":\"{\\\"targetPath\\\":\\\"/ods/tmp/test/person7\\\",\\\"deleteTargetDir\\\":true,\\\"fileType\\\":\\\"--as-textfile\\\",\\\"compressionCodec\\\":\\\"\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}";
SqoopParameters sqoopParameters1 = JSONObject.parseObject(data1,SqoopParameters.class); SqoopParameters sqoopParameters1 = JSON.parseObject(data1,SqoopParameters.class);
SqoopJobGenerator generator = new SqoopJobGenerator(); SqoopJobGenerator generator = new SqoopJobGenerator();
String script = generator.generateSqoopJob(sqoopParameters1); String script = generator.generateSqoopJob(sqoopParameters1);
@ -82,21 +82,21 @@ public class SqoopTaskTest {
Assert.assertEquals(expected, script); Assert.assertEquals(expected, script);
String data2 = "{\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HDFS\",\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"exportDir\\\":\\\"/ods/tmp/test/person7\\\"}\",\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"id,name,age,sex,create_time\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":true,\\\"targetUpdateKey\\\":\\\"id\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; String data2 = "{\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HDFS\",\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"exportDir\\\":\\\"/ods/tmp/test/person7\\\"}\",\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"id,name,age,sex,create_time\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":true,\\\"targetUpdateKey\\\":\\\"id\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}";
SqoopParameters sqoopParameters2 = JSONObject.parseObject(data2,SqoopParameters.class); SqoopParameters sqoopParameters2 = JSON.parseObject(data2,SqoopParameters.class);
String script2 = generator.generateSqoopJob(sqoopParameters2); String script2 = generator.generateSqoopJob(sqoopParameters2);
String expected2 = "sqoop export -m 1 --export-dir /ods/tmp/test/person7 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --columns id,name,age,sex,create_time --fields-terminated-by '@' --lines-terminated-by '\\n' --update-key id --update-mode allowinsert"; String expected2 = "sqoop export -m 1 --export-dir /ods/tmp/test/person7 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --columns id,name,age,sex,create_time --fields-terminated-by '@' --lines-terminated-by '\\n' --update-key id --update-mode allowinsert";
Assert.assertEquals(expected2, script2); Assert.assertEquals(expected2, script2);
String data3 = "{\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HIVE\",\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-17\\\"}\",\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":false,\\\"targetUpdateKey\\\":\\\"\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; String data3 = "{\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HIVE\",\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-17\\\"}\",\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":false,\\\"targetUpdateKey\\\":\\\"\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}";
SqoopParameters sqoopParameters3 = JSONObject.parseObject(data3,SqoopParameters.class); SqoopParameters sqoopParameters3 = JSON.parseObject(data3,SqoopParameters.class);
String script3 = generator.generateSqoopJob(sqoopParameters3); String script3 = generator.generateSqoopJob(sqoopParameters3);
String expected3 = "sqoop export -m 1 --hcatalog-database stg --hcatalog-table person_internal --hcatalog-partition-keys date --hcatalog-partition-values 2020-02-17 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --fields-terminated-by '@' --lines-terminated-by '\\n'"; String expected3 = "sqoop export -m 1 --hcatalog-database stg --hcatalog-table person_internal --hcatalog-partition-keys date --hcatalog-partition-values 2020-02-17 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --fields-terminated-by '@' --lines-terminated-by '\\n'";
Assert.assertEquals(expected3, script3); Assert.assertEquals(expected3, script3);
String data4 = "{\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HIVE\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"1\\\",\\\"srcQuerySql\\\":\\\"SELECT * FROM person_2\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[{\\\"prop\\\":\\\"id\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"Integer\\\"}]}\",\"targetParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal_2\\\",\\\"createHiveTable\\\":true,\\\"dropDelimiter\\\":false,\\\"hiveOverWrite\\\":true,\\\"replaceDelimiter\\\":\\\"\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-16\\\"}\",\"localParams\":[]}"; String data4 = "{\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HIVE\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"1\\\",\\\"srcQuerySql\\\":\\\"SELECT * FROM person_2\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[{\\\"prop\\\":\\\"id\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"Integer\\\"}]}\",\"targetParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal_2\\\",\\\"createHiveTable\\\":true,\\\"dropDelimiter\\\":false,\\\"hiveOverWrite\\\":true,\\\"replaceDelimiter\\\":\\\"\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-16\\\"}\",\"localParams\":[]}";
SqoopParameters sqoopParameters4 = JSONObject.parseObject(data4,SqoopParameters.class); SqoopParameters sqoopParameters4 = JSON.parseObject(data4,SqoopParameters.class);
String script4 = generator.generateSqoopJob(sqoopParameters4); String script4 = generator.generateSqoopJob(sqoopParameters4);
String expected4 = "sqoop import -m 1 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --query 'SELECT * FROM person_2 WHERE $CONDITIONS' --map-column-java id=Integer --hive-import --hive-table stg.person_internal_2 --create-hive-table --hive-overwrite -delete-target-dir --hive-partition-key date --hive-partition-value 2020-02-16"; String expected4 = "sqoop import -m 1 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --query 'SELECT * FROM person_2 WHERE $CONDITIONS' --map-column-java id=Integer --hive-import --hive-table stg.person_internal_2 --create-hive-table --hive-overwrite -delete-target-dir --hive-partition-key date --hive-partition-value 2020-02-16";

7
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
Unescape View File

@ -16,6 +16,7 @@
*/ */
package org.apache.dolphinscheduler.service.process; package org.apache.dolphinscheduler.service.process;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.cronutils.model.Cron; import com.cronutils.model.Cron;
import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.ArrayUtils;
@ -207,7 +208,7 @@ public class ProcessService {
CommandType commandType = command.getCommandType(); CommandType commandType = command.getCommandType();
if(cmdTypeMap.containsKey(commandType)){ if(cmdTypeMap.containsKey(commandType)){
JSONObject cmdParamObj = (JSONObject) JSONObject.parse(command.getCommandParam()); JSONObject cmdParamObj = (JSONObject) JSON.parse(command.getCommandParam());
JSONObject tempObj; JSONObject tempObj;
int processInstanceId = cmdParamObj.getInteger(CMDPARAM_RECOVER_PROCESS_ID_STRING); int processInstanceId = cmdParamObj.getInteger(CMDPARAM_RECOVER_PROCESS_ID_STRING);
@ -215,7 +216,7 @@ public class ProcessService {
// for all commands // for all commands
for (Command tmpCommand:commands){ for (Command tmpCommand:commands){
if(cmdTypeMap.containsKey(tmpCommand.getCommandType())){ if(cmdTypeMap.containsKey(tmpCommand.getCommandType())){
tempObj = (JSONObject) JSONObject.parse(tmpCommand.getCommandParam()); tempObj = (JSONObject) JSON.parse(tmpCommand.getCommandParam());
if(tempObj != null && processInstanceId == tempObj.getInteger(CMDPARAM_RECOVER_PROCESS_ID_STRING)){ if(tempObj != null && processInstanceId == tempObj.getInteger(CMDPARAM_RECOVER_PROCESS_ID_STRING)){
isNeedCreate = false; isNeedCreate = false;
break; break;
@ -309,7 +310,7 @@ public class ProcessService {
for (TaskNode taskNode : taskNodeList){ for (TaskNode taskNode : taskNodeList){
String parameter = taskNode.getParams(); String parameter = taskNode.getParams();
if (parameter.contains(CMDPARAM_SUB_PROCESS_DEFINE_ID)){ if (parameter.contains(CMDPARAM_SUB_PROCESS_DEFINE_ID)){
SubProcessParameters subProcessParam = JSONObject.parseObject(parameter, SubProcessParameters.class); SubProcessParameters subProcessParam = JSON.parseObject(parameter, SubProcessParameters.class);
ids.add(subProcessParam.getProcessDefinitionId()); ids.add(subProcessParam.getProcessDefinitionId());
recurseFindSubProcessId(subProcessParam.getProcessDefinitionId(),ids); recurseFindSubProcessId(subProcessParam.getProcessDefinitionId(),ids);
} }

Write Preview
Loading…
Cancel
Save