From b49b91c45d1c9e1ed1906b49d1fef63d753ca1ac Mon Sep 17 00:00:00 2001 From: Rick Cheng Date: Wed, 14 Jun 2023 15:00:54 +0800 Subject: [PATCH] [Improvement-14331][common] Remove the duplicate ParameterUtils (#14332) --- .../runner/StreamTaskExecuteRunnable.java | 4 +- .../runner/task/dynamic/DynamicLogicTask.java | 5 +- .../task/switchtask/SwitchLogicTask.java | 2 +- .../service/expand/CuringGlobalParams.java | 7 +- .../plugin/task/api/parser/ParamUtils.java | 71 ---- .../task/api/parser/ParameterUtils.java | 306 ------------------ .../plugin/task/api/utils/ParameterUtils.java | 241 +++++++++++++- .../task/api/parser/ParameterUtilsTest.java | 55 ---- .../api/parser/TimePlaceholderUtilsTest.java | 1 + .../task/api/utils/ParameterUtilsTest.java | 36 ++- .../plugin/task/chunjun/ChunJunTask.java | 7 +- .../plugin/task/dq/DataQualityTask.java | 5 +- .../MultiTableComparisonRuleParser.java | 2 +- .../plugin/task/dq/utils/RuleParserUtils.java | 2 +- .../plugin/task/datax/DataxTask.java | 7 +- .../plugin/task/dms/DmsTask.java | 5 +- .../plugin/task/flink/FlinkStreamTask.java | 2 +- .../plugin/task/flink/FlinkArgsUtils.java | 5 +- .../plugin/task/flink/FlinkTask.java | 2 +- .../plugin/task/hivecli/HiveCliTask.java | 5 +- .../plugin/task/http/HttpTask.java | 8 +- .../plugin/task/java/JavaTask.java | 5 +- .../plugin/task/jupyter/JupyterTask.java | 5 +- .../plugin/task/k8s/K8sTask.java | 4 +- .../plugin/kubeflow/KubeflowTask.java | 5 +- .../plugin/task/linkis/LinkisTask.java | 5 +- .../plugin/task/mlflow/MlflowTask.java | 7 +- .../plugin/task/mr/MapReduceTask.java | 7 +- .../plugin/task/openmldb/OpenmldbTask.java | 5 +- .../plugin/task/procedure/ProcedureTask.java | 2 +- .../plugin/task/python/PythonTask.java | 5 +- .../plugin/task/pytorch/PytorchTask.java | 5 +- .../task/remoteshell/RemoteShellTask.java | 5 +- .../plugin/task/sagemaker/SagemakerTask.java | 5 +- .../plugin/task/seatunnel/SeatunnelTask.java | 5 +- .../plugin/task/shell/ShellTask.java | 5 +- .../plugin/task/spark/SparkTask.java | 7 +- .../plugin/task/sql/SqlTask.java | 5 +- .../plugin/task/sqoop/SqoopTask.java | 5 +- 39 files changed, 322 insertions(+), 548 deletions(-) delete mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParamUtils.java delete mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParameterUtils.java delete mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParameterUtilsTest.java diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java index b90d57a343..5f64a3c099 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java @@ -40,8 +40,8 @@ import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.remote.command.task.TaskExecuteRunningMessageAck; import org.apache.dolphinscheduler.remote.command.task.TaskExecuteStartMessage; import org.apache.dolphinscheduler.server.master.builder.TaskExecutionContextBuilder; @@ -436,7 +436,7 @@ public class StreamTaskExecuteRunnable implements Runnable { @NonNull AbstractParameters parameters) { // assign value to definedParams here Map globalParamsMap = taskExecuteStartMessage.getStartParams(); - Map globalParams = ParamUtils.getUserDefParamsMap(globalParamsMap); + Map globalParams = ParameterUtils.getUserDefParamsMap(globalParamsMap); // combining local and global parameters Map localParams = parameters.getInputLocalParametersMap(); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTask.java index 377226dcb8..453446fda2 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTask.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTask.java @@ -34,8 +34,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.DynamicInputParameter; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.DynamicParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.remote.command.workflow.WorkflowStateEventChangeRequest; import org.apache.dolphinscheduler.remote.exceptions.RemotingException; import org.apache.dolphinscheduler.remote.utils.Host; @@ -248,7 +247,7 @@ public class DynamicLogicTask extends BaseAsyncLogicTask { for (DynamicInputParameter dynamicInputParameter : dynamicInputParameters) { String value = dynamicInputParameter.getValue(); Map paramsMap = taskExecutionContext.getPrepareParamsMap(); - value = ParameterUtils.convertParameterPlaceholders(value, ParamUtils.convert(paramsMap)); + value = ParameterUtils.convertParameterPlaceholders(value, ParameterUtils.convert(paramsMap)); dynamicInputParameter.setValue(value); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/switchtask/SwitchLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/switchtask/SwitchLogicTask.java index 038cff7797..c6e20950e0 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/switchtask/SwitchLogicTask.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/switchtask/SwitchLogicTask.java @@ -26,7 +26,7 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.SwitchResultVo; import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException; import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException; diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringGlobalParams.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringGlobalParams.java index a916f4ad35..28f8f4c087 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringGlobalParams.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringGlobalParams.java @@ -37,7 +37,6 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; @@ -143,7 +142,7 @@ public class CuringGlobalParams implements CuringParamsService { @NonNull ProcessInstance processInstance) { // assign value to definedParams here Map globalParamsMap = setGlobalParamsMap(processInstance); - Map globalParams = ParamUtils.getUserDefParamsMap(globalParamsMap); + Map globalParams = ParameterUtils.getUserDefParamsMap(globalParamsMap); // combining local and global parameters Map localParams = parameters.getInputLocalParametersMap(); @@ -162,7 +161,7 @@ public class CuringGlobalParams implements CuringParamsService { Map params = setBuiltInParamsMap(taskInstance, timeZone); if (MapUtils.isNotEmpty(params)) { - globalParams.putAll(ParamUtils.getUserDefParamsMap(params)); + globalParams.putAll(ParameterUtils.getUserDefParamsMap(params)); } if (MapUtils.isNotEmpty(varParams)) { @@ -172,7 +171,7 @@ public class CuringGlobalParams implements CuringParamsService { globalParams.putAll(localParams); } if (MapUtils.isNotEmpty(cmdParam)) { - globalParams.putAll(ParamUtils.getUserDefParamsMap(cmdParam)); + globalParams.putAll(ParameterUtils.getUserDefParamsMap(cmdParam)); } Iterator> iter = globalParams.entrySet().iterator(); while (iter.hasNext()) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParamUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParamUtils.java deleted file mode 100644 index c106f7932d..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParamUtils.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.api.parser; - -import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; -import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; -import org.apache.dolphinscheduler.plugin.task.api.model.Property; - -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -/** - * param utils - */ -public class ParamUtils { - - /** - * format convert - * - * @param paramsMap params map - * @return Map of converted - */ - public static Map convert(Map paramsMap) { - if (paramsMap == null) { - return null; - } - - Map map = new HashMap<>(); - Iterator> iter = paramsMap.entrySet().iterator(); - while (iter.hasNext()) { - Map.Entry en = iter.next(); - map.put(en.getKey(), en.getValue().getValue()); - } - return map; - } - - /** - * get parameters map - * - * @param definedParams definedParams - * @return parameters map - */ - public static Map getUserDefParamsMap(Map definedParams) { - Map userDefParamsMaps = new HashMap<>(); - if (definedParams != null) { - Iterator> iter = definedParams.entrySet().iterator(); - while (iter.hasNext()) { - Map.Entry en = iter.next(); - Property property = new Property(en.getKey(), Direct.IN, DataType.VARCHAR, en.getValue()); - userDefParamsMaps.put(property.getProp(), property); - } - } - return userDefParamsMaps; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParameterUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParameterUtils.java deleted file mode 100644 index 920b635572..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParameterUtils.java +++ /dev/null @@ -1,306 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.api.parser; - -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_DATETIME; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_FORMAT_TIME; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_SHECDULE_TIME; - -import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; -import org.apache.dolphinscheduler.plugin.task.api.model.Property; - -import org.apache.commons.lang3.StringUtils; - -import java.io.Serializable; -import java.sql.PreparedStatement; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * parameter parse utils - */ -public class ParameterUtils { - - private static final Pattern DATE_PARSE_PATTERN = Pattern.compile("\\$\\[([^\\$\\]]+)]"); - - private static final Pattern DATE_START_PATTERN = Pattern.compile("^[0-9]"); - - private static final char PARAM_REPLACE_CHAR = '?'; - - private ParameterUtils() { - throw new UnsupportedOperationException("Construct ParameterUtils"); - } - - /** - * convert parameters place holders - * - * @param parameterString parameter - * @param parameterMap parameter map - * @return convert parameters place holders - */ - public static String convertParameterPlaceholders(String parameterString, Map parameterMap) { - if (StringUtils.isEmpty(parameterString)) { - return parameterString; - } - Date cronTime; - if (parameterMap != null && !parameterMap.isEmpty()) { - // replace variable ${} form,refers to the replacement of system variables and custom variables - parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true); - } - if (parameterMap != null && null != parameterMap.get(PARAMETER_DATETIME)) { - // Get current time, schedule execute time - String cronTimeStr = parameterMap.get(PARAMETER_DATETIME); - cronTime = DateUtils.parse(cronTimeStr, PARAMETER_FORMAT_TIME); - } else { - cronTime = new Date(); - } - // replace time $[...] form, eg. $[yyyyMMdd] - if (cronTime != null) { - return dateTemplateParse(parameterString, cronTime); - } - return parameterString; - } - - /** - * new - * convert parameters place holders - * - * @param parameterString parameter - * @param parameterMap parameter map - * @return convert parameters place holders - */ - public static String convertParameterPlaceholders2(String parameterString, Map parameterMap) { - if (StringUtils.isEmpty(parameterString)) { - return parameterString; - } - // Get current time, schedule execute time - String cronTimeStr = parameterMap.get(PARAMETER_SHECDULE_TIME); - Date cronTime = null; - - if (StringUtils.isNotEmpty(cronTimeStr)) { - cronTime = DateUtils.parse(cronTimeStr, PARAMETER_FORMAT_TIME); - - } else { - cronTime = new Date(); - } - - // replace variable ${} form,refers to the replacement of system variables and custom variables - if (!parameterMap.isEmpty()) { - parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true); - } - - // replace time $[...] form, eg. $[yyyyMMdd] - if (cronTime != null) { - return dateTemplateParse(parameterString, cronTime); - } - return parameterString; - } - - /** - * set in parameter - * - * @param index index - * @param stmt preparedstatement - * @param dataType data type - * @param value value - * @throws Exception errors - */ - public static void setInParameter(int index, PreparedStatement stmt, DataType dataType, - String value) throws Exception { - if (dataType.equals(DataType.VARCHAR)) { - stmt.setString(index, value); - } else if (dataType.equals(DataType.INTEGER)) { - stmt.setInt(index, Integer.parseInt(value)); - } else if (dataType.equals(DataType.LONG)) { - stmt.setLong(index, Long.parseLong(value)); - } else if (dataType.equals(DataType.FLOAT)) { - stmt.setFloat(index, Float.parseFloat(value)); - } else if (dataType.equals(DataType.DOUBLE)) { - stmt.setDouble(index, Double.parseDouble(value)); - } else if (dataType.equals(DataType.DATE)) { - stmt.setDate(index, java.sql.Date.valueOf(value)); - } else if (dataType.equals(DataType.TIME)) { - stmt.setString(index, value); - } else if (dataType.equals(DataType.TIMESTAMP)) { - stmt.setTimestamp(index, java.sql.Timestamp.valueOf(value)); - } else if (dataType.equals(DataType.BOOLEAN)) { - stmt.setBoolean(index, Boolean.parseBoolean(value)); - } - } - - public static Serializable getParameterValue(Property property) { - if (property == null) { - return null; - } - String value = property.getValue(); - switch (property.getType()) { - case LONG: - return Long.valueOf(value); - case FLOAT: - return Float.valueOf(value); - case INTEGER: - return Integer.valueOf(value); - case DOUBLE: - return Double.valueOf(value); - case BOOLEAN: - return Boolean.valueOf(value); - // todo: add date type, list type.... - default: - return value; - } - } - - public static boolean isNumber(Property property) { - return property != null && - (DataType.INTEGER.equals(property.getType()) - || DataType.LONG.equals(property.getType()) - || DataType.FLOAT.equals(property.getType()) - || DataType.DOUBLE.equals(property.getType())); - } - - public static boolean isBoolean(Property property) { - return property != null && DataType.BOOLEAN.equals(property.getType()); - } - - public static String expandListParameter(Map params, String sql) { - Map expandMap = new HashMap<>(); - if (params == null || params.isEmpty()) { - return sql; - } - String[] split = sql.split("\\?"); - if (split.length == 0) { - return sql; - } - StringBuilder ret = new StringBuilder(split[0]); - int index = 1; - for (int i = 1; i < split.length; i++) { - Property property = params.get(i); - String value = property.getValue(); - if (DataType.LIST.equals(property.getType())) { - List valueList = JSONUtils.toList(value, Object.class); - if (valueList.isEmpty() && StringUtils.isNotBlank(value)) { - valueList.add(value); - } - for (int j = 0; j < valueList.size(); j++) { - ret.append(PARAM_REPLACE_CHAR); - if (j != valueList.size() - 1) { - ret.append(","); - } - } - for (Object v : valueList) { - Property newProperty = new Property(); - if (v instanceof Integer) { - newProperty.setType(DataType.INTEGER); - } else if (v instanceof Long) { - newProperty.setType(DataType.LONG); - } else if (v instanceof Float) { - newProperty.setType(DataType.FLOAT); - } else if (v instanceof Double) { - newProperty.setType(DataType.DOUBLE); - } else { - newProperty.setType(DataType.VARCHAR); - } - newProperty.setValue(v.toString()); - newProperty.setProp(property.getProp()); - newProperty.setDirect(property.getDirect()); - expandMap.put(index++, newProperty); - } - } else { - ret.append(PARAM_REPLACE_CHAR); - expandMap.put(index++, property); - } - ret.append(split[i]); - } - if (PARAM_REPLACE_CHAR == sql.charAt(sql.length() - 1)) { - ret.append(PARAM_REPLACE_CHAR); - expandMap.put(index, params.get(split.length)); - } - params.clear(); - params.putAll(expandMap); - return ret.toString(); - } - - /** - * $[yyyyMMdd] replace schedule time - */ - public static String replaceScheduleTime(String text, Date scheduleTime) { - Map paramsMap = new HashMap<>(); - // if getScheduleTime null ,is current date - if (null == scheduleTime) { - scheduleTime = new Date(); - } - - String dateTime = DateUtils.format(scheduleTime, PARAMETER_FORMAT_TIME); - Property p = new Property(); - p.setValue(dateTime); - p.setProp(PARAMETER_SHECDULE_TIME); - paramsMap.put(PARAMETER_SHECDULE_TIME, p); - text = ParameterUtils.convertParameterPlaceholders2(text, convert(paramsMap)); - - return text; - } - - /** - * format convert - * - * @param paramsMap params map - * @return Map of converted - * see org.apache.dolphinscheduler.server.utils.ParamUtils.convert - */ - public static Map convert(Map paramsMap) { - Map map = new HashMap<>(); - Iterator> iter = paramsMap.entrySet().iterator(); - while (iter.hasNext()) { - Map.Entry en = iter.next(); - map.put(en.getKey(), en.getValue().getValue()); - } - return map; - } - - private static String dateTemplateParse(String templateStr, Date date) { - if (templateStr == null) { - return null; - } - - StringBuffer newValue = new StringBuffer(templateStr.length()); - - Matcher matcher = DATE_PARSE_PATTERN.matcher(templateStr); - - while (matcher.find()) { - String key = matcher.group(1); - if (DATE_START_PATTERN.matcher(key).matches()) { - continue; - } - String value = TimePlaceholderUtils.getPlaceHolderTime(key, date); - assert value != null; - matcher.appendReplacement(newValue, value); - } - - matcher.appendTail(newValue); - - return newValue.toString(); - } - -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java index 781209324a..11d1bb113c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java @@ -17,19 +17,27 @@ package org.apache.dolphinscheduler.plugin.task.api.utils; -import org.apache.dolphinscheduler.common.constants.DateConstants; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_DATETIME; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_FORMAT_TIME; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_SHECDULE_TIME; + import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; +import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parser.PlaceholderUtils; import org.apache.dolphinscheduler.plugin.task.api.parser.TimePlaceholderUtils; import org.apache.commons.lang3.StringUtils; +import java.io.Serializable; +import java.sql.PreparedStatement; import java.util.Date; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -39,8 +47,11 @@ import java.util.regex.Pattern; public class ParameterUtils { private static final Pattern DATE_PARSE_PATTERN = Pattern.compile("\\$\\[([^\\$\\]]+)]"); + private static final Pattern DATE_START_PATTERN = Pattern.compile("^[0-9]"); + private static final char PARAM_REPLACE_CHAR = '?'; + private ParameterUtils() { throw new UnsupportedOperationException("Construct ParameterUtils"); } @@ -61,10 +72,10 @@ public class ParameterUtils { // replace variable ${} form,refers to the replacement of system variables and custom variables parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true); } - if (parameterMap != null && null != parameterMap.get(DateConstants.PARAMETER_DATETIME)) { + if (parameterMap != null && null != parameterMap.get(PARAMETER_DATETIME)) { // Get current time, schedule execute time - String cronTimeStr = parameterMap.get(DateConstants.PARAMETER_DATETIME); - cronTime = DateUtils.parse(cronTimeStr, DateConstants.PARAMETER_FORMAT_TIME, null); + String cronTimeStr = parameterMap.get(PARAMETER_DATETIME); + cronTime = DateUtils.parse(cronTimeStr, PARAMETER_FORMAT_TIME); } else { cronTime = new Date(); } @@ -76,17 +87,182 @@ public class ParameterUtils { } /** - * handle escapes + * new + * convert parameters place holders * - * @param inputString input string - * @return string filter escapes + * @param parameterString parameter + * @param parameterMap parameter map + * @return convert parameters place holders */ - public static String handleEscapes(String inputString) { + public static String convertParameterPlaceholders2(String parameterString, Map parameterMap) { + if (StringUtils.isEmpty(parameterString)) { + return parameterString; + } + // Get current time, schedule execute time + String cronTimeStr = parameterMap.get(PARAMETER_SHECDULE_TIME); + Date cronTime = null; - if (!StringUtils.isEmpty(inputString)) { - return inputString.replace("%", "////%").replaceAll("[\n|\r\t]", "_"); + if (StringUtils.isNotEmpty(cronTimeStr)) { + cronTime = DateUtils.parse(cronTimeStr, PARAMETER_FORMAT_TIME); + + } else { + cronTime = new Date(); } - return inputString; + + // replace variable ${} form,refers to the replacement of system variables and custom variables + if (!parameterMap.isEmpty()) { + parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true); + } + + // replace time $[...] form, eg. $[yyyyMMdd] + if (cronTime != null) { + return dateTemplateParse(parameterString, cronTime); + } + return parameterString; + } + + /** + * set in parameter + * + * @param index index + * @param stmt preparedstatement + * @param dataType data type + * @param value value + * @throws Exception errors + */ + public static void setInParameter(int index, PreparedStatement stmt, DataType dataType, + String value) throws Exception { + if (dataType.equals(DataType.VARCHAR)) { + stmt.setString(index, value); + } else if (dataType.equals(DataType.INTEGER)) { + stmt.setInt(index, Integer.parseInt(value)); + } else if (dataType.equals(DataType.LONG)) { + stmt.setLong(index, Long.parseLong(value)); + } else if (dataType.equals(DataType.FLOAT)) { + stmt.setFloat(index, Float.parseFloat(value)); + } else if (dataType.equals(DataType.DOUBLE)) { + stmt.setDouble(index, Double.parseDouble(value)); + } else if (dataType.equals(DataType.DATE)) { + stmt.setDate(index, java.sql.Date.valueOf(value)); + } else if (dataType.equals(DataType.TIME)) { + stmt.setString(index, value); + } else if (dataType.equals(DataType.TIMESTAMP)) { + stmt.setTimestamp(index, java.sql.Timestamp.valueOf(value)); + } else if (dataType.equals(DataType.BOOLEAN)) { + stmt.setBoolean(index, Boolean.parseBoolean(value)); + } + } + + public static Serializable getParameterValue(Property property) { + if (property == null) { + return null; + } + String value = property.getValue(); + switch (property.getType()) { + case LONG: + return Long.valueOf(value); + case FLOAT: + return Float.valueOf(value); + case INTEGER: + return Integer.valueOf(value); + case DOUBLE: + return Double.valueOf(value); + case BOOLEAN: + return Boolean.valueOf(value); + // todo: add date type, list type.... + default: + return value; + } + } + + public static boolean isNumber(Property property) { + return property != null && + (DataType.INTEGER.equals(property.getType()) + || DataType.LONG.equals(property.getType()) + || DataType.FLOAT.equals(property.getType()) + || DataType.DOUBLE.equals(property.getType())); + } + + public static boolean isBoolean(Property property) { + return property != null && DataType.BOOLEAN.equals(property.getType()); + } + + public static String expandListParameter(Map params, String sql) { + Map expandMap = new HashMap<>(); + if (params == null || params.isEmpty()) { + return sql; + } + String[] split = sql.split("\\?"); + if (split.length == 0) { + return sql; + } + StringBuilder ret = new StringBuilder(split[0]); + int index = 1; + for (int i = 1; i < split.length; i++) { + Property property = params.get(i); + String value = property.getValue(); + if (DataType.LIST.equals(property.getType())) { + List valueList = JSONUtils.toList(value, Object.class); + if (valueList.isEmpty() && StringUtils.isNotBlank(value)) { + valueList.add(value); + } + for (int j = 0; j < valueList.size(); j++) { + ret.append(PARAM_REPLACE_CHAR); + if (j != valueList.size() - 1) { + ret.append(","); + } + } + for (Object v : valueList) { + Property newProperty = new Property(); + if (v instanceof Integer) { + newProperty.setType(DataType.INTEGER); + } else if (v instanceof Long) { + newProperty.setType(DataType.LONG); + } else if (v instanceof Float) { + newProperty.setType(DataType.FLOAT); + } else if (v instanceof Double) { + newProperty.setType(DataType.DOUBLE); + } else { + newProperty.setType(DataType.VARCHAR); + } + newProperty.setValue(v.toString()); + newProperty.setProp(property.getProp()); + newProperty.setDirect(property.getDirect()); + expandMap.put(index++, newProperty); + } + } else { + ret.append(PARAM_REPLACE_CHAR); + expandMap.put(index++, property); + } + ret.append(split[i]); + } + if (PARAM_REPLACE_CHAR == sql.charAt(sql.length() - 1)) { + ret.append(PARAM_REPLACE_CHAR); + expandMap.put(index, params.get(split.length)); + } + params.clear(); + params.putAll(expandMap); + return ret.toString(); + } + + /** + * $[yyyyMMdd] replace schedule time + */ + public static String replaceScheduleTime(String text, Date scheduleTime) { + Map paramsMap = new HashMap<>(); + // if getScheduleTime null ,is current date + if (null == scheduleTime) { + scheduleTime = new Date(); + } + + String dateTime = DateUtils.format(scheduleTime, PARAMETER_FORMAT_TIME); + Property p = new Property(); + p.setValue(dateTime); + p.setProp(PARAMETER_SHECDULE_TIME); + paramsMap.put(PARAMETER_SHECDULE_TIME, p); + text = ParameterUtils.convertParameterPlaceholders2(text, convert(paramsMap)); + + return text; } /** @@ -94,11 +270,15 @@ public class ParameterUtils { * * @param paramsMap params map * @return Map of converted - * see org.apache.dolphinscheduler.server.utils.ParamUtils.convert + * see org.apache.dolphinscheduler.server.utils.ParameterUtils.convert */ public static Map convert(Map paramsMap) { + if (paramsMap == null) { + return null; + } + Map map = new HashMap<>(); - Iterator> iter = paramsMap.entrySet().iterator(); + Iterator> iter = paramsMap.entrySet().iterator(); while (iter.hasNext()) { Map.Entry en = iter.next(); map.put(en.getKey(), en.getValue().getValue()); @@ -130,4 +310,37 @@ public class ParameterUtils { return newValue.toString(); } + /** + * handle escapes + * + * @param inputString input string + * @return string filter escapes + */ + public static String handleEscapes(String inputString) { + + if (!StringUtils.isEmpty(inputString)) { + return inputString.replace("%", "////%").replaceAll("[\n|\r\t]", "_"); + } + return inputString; + } + + /** + * get parameters map + * + * @param definedParams definedParams + * @return parameters map + */ + public static Map getUserDefParamsMap(Map definedParams) { + Map userDefParamsMaps = new HashMap<>(); + if (definedParams != null) { + Iterator> iter = definedParams.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry en = iter.next(); + Property property = new Property(en.getKey(), Direct.IN, DataType.VARCHAR, en.getValue()); + userDefParamsMaps.put(property.getProp(), property); + } + } + return userDefParamsMaps; + } + } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParameterUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParameterUtilsTest.java deleted file mode 100644 index 12f1f68276..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParameterUtilsTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.api.parser; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; -import org.apache.dolphinscheduler.plugin.task.api.model.Property; - -import java.util.HashMap; -import java.util.Map; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import com.google.common.collect.Lists; - -public class ParameterUtilsTest { - - @Test - public void expandListParameter() { - Map params = new HashMap<>(); - params.put(1, - new Property(null, null, DataType.LIST, JSONUtils.toJsonString(Lists.newArrayList("c1", "c2", "c3")))); - params.put(2, new Property(null, null, DataType.DATE, "2020-06-30")); - params.put(3, new Property(null, null, DataType.LIST, - JSONUtils.toJsonString(Lists.newArrayList(3.1415, 2.44, 3.44)))); - String sql = ParameterUtils.expandListParameter(params, - "select * from test where col1 in (?) and date=? and col2 in (?)"); - Assertions.assertEquals("select * from test where col1 in (?,?,?) and date=? and col2 in (?,?,?)", sql); - Assertions.assertEquals(7, params.size()); - - Map params2 = new HashMap<>(); - params2.put(1, new Property(null, null, DataType.LIST, JSONUtils.toJsonString(Lists.newArrayList("c1")))); - params2.put(2, new Property(null, null, DataType.DATE, "2020-06-30")); - String sql2 = ParameterUtils.expandListParameter(params2, "select * from test where col1 in (?) and date=?"); - Assertions.assertEquals("select * from test where col1 in (?) and date=?", sql2); - Assertions.assertEquals(2, params2.size()); - - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtilsTest.java index dc8e16b9e3..448f45fd55 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtilsTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtilsTest.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.plugin.task.api.parser; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import java.util.Date; import java.util.Map; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java index b1189b689e..4297e36fcc 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java @@ -21,6 +21,9 @@ import static org.apache.dolphinscheduler.plugin.task.api.parser.TimePlaceholder import org.apache.dolphinscheduler.common.constants.DateConstants; import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parser.PlaceholderUtils; import java.text.ParseException; @@ -30,19 +33,33 @@ import java.util.Map; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -@ExtendWith(MockitoExtension.class) +import com.google.common.collect.Lists; + public class ParameterUtilsTest { - public static final Logger logger = LoggerFactory.getLogger(ParameterUtilsTest.class); + @Test + public void expandListParameter() { + Map params = new HashMap<>(); + params.put(1, + new Property(null, null, DataType.LIST, JSONUtils.toJsonString(Lists.newArrayList("c1", "c2", "c3")))); + params.put(2, new Property(null, null, DataType.DATE, "2020-06-30")); + params.put(3, new Property(null, null, DataType.LIST, + JSONUtils.toJsonString(Lists.newArrayList(3.1415, 2.44, 3.44)))); + String sql = ParameterUtils.expandListParameter(params, + "select * from test where col1 in (?) and date=? and col2 in (?)"); + Assertions.assertEquals("select * from test where col1 in (?,?,?) and date=? and col2 in (?,?,?)", sql); + Assertions.assertEquals(7, params.size()); + + Map params2 = new HashMap<>(); + params2.put(1, new Property(null, null, DataType.LIST, JSONUtils.toJsonString(Lists.newArrayList("c1")))); + params2.put(2, new Property(null, null, DataType.DATE, "2020-06-30")); + String sql2 = ParameterUtils.expandListParameter(params2, "select * from test where col1 in (?) and date=?"); + Assertions.assertEquals("select * from test where col1 in (?) and date=?", sql2); + Assertions.assertEquals(2, params2.size()); + + } - /** - * Test convertParameterPlaceholders - */ @Test public void testConvertParameterPlaceholders() throws ParseException { // parameterString,parameterMap is null @@ -96,5 +113,4 @@ public class ParameterUtilsTest { Assertions.assertEquals("test Parameter", ParameterUtils.handleEscapes("test Parameter")); Assertions.assertEquals("////%test////%Parameter", ParameterUtils.handleEscapes("%test%Parameter")); } - } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTask.java index 7f09243104..92bd3ea69e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTask.java @@ -30,8 +30,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.spi.enums.Flag; import org.apache.commons.io.FileUtils; @@ -159,7 +158,7 @@ public class ChunJunTask extends AbstractTask { } // replace placeholder - json = ParameterUtils.convertParameterPlaceholders(json, ParamUtils.convert(paramsMap)); + json = ParameterUtils.convertParameterPlaceholders(json, ParameterUtils.convert(paramsMap)); log.debug("chunjun job json : {}", json); @@ -217,7 +216,7 @@ public class ChunJunTask extends AbstractTask { String command = String.join(" ", args); // replace placeholder - String chunjunCommand = ParameterUtils.convertParameterPlaceholders(command, ParamUtils.convert(paramsMap)); + String chunjunCommand = ParameterUtils.convertParameterPlaceholders(command, ParameterUtils.convert(paramsMap)); log.info("raw script : {}", chunjunCommand); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTask.java index b270b9a5f5..95617b5a88 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTask.java @@ -42,9 +42,8 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.DataQualityParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ArgsUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.dq.rule.RuleManager; import org.apache.dolphinscheduler.plugin.task.dq.rule.parameter.DataQualityConfiguration; import org.apache.dolphinscheduler.plugin.task.dq.utils.SparkArgsUtils; @@ -174,7 +173,7 @@ public class DataQualityTask extends AbstractYarnTask { // replace placeholder Map paramsMap = dqTaskExecutionContext.getPrepareParamsMap(); String command = - ParameterUtils.convertParameterPlaceholders(String.join(" ", args), ParamUtils.convert(paramsMap)); + ParameterUtils.convertParameterPlaceholders(String.join(" ", args), ParameterUtils.convert(paramsMap)); log.info("data quality task command: {}", command); return command; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableComparisonRuleParser.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableComparisonRuleParser.java index b6a130366b..27a1f83838 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableComparisonRuleParser.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableComparisonRuleParser.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.plugin.task.dq.rule.parser; import org.apache.dolphinscheduler.plugin.task.api.DataQualityTaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.dq.exception.DataQualityException; import org.apache.dolphinscheduler.plugin.task.dq.rule.RuleManager; import org.apache.dolphinscheduler.plugin.task.dq.rule.parameter.BaseConfig; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java index e8bc50ab40..ce248d2cdf 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java @@ -64,8 +64,8 @@ import org.apache.dolphinscheduler.data.quality.utils.ParserUtils; import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; import org.apache.dolphinscheduler.plugin.task.api.DataQualityTaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.dp.ExecuteSqlType; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.dq.exception.DataQualityException; import org.apache.dolphinscheduler.plugin.task.dq.rule.entity.DqRuleExecuteSql; import org.apache.dolphinscheduler.plugin.task.dq.rule.entity.DqRuleInputEntry; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java index 129e656943..4477b42eb7 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java @@ -33,8 +33,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.dolphinscheduler.spi.enums.Flag; @@ -230,7 +229,7 @@ public class DataxTask extends AbstractTask { } // replace placeholder - json = ParameterUtils.convertParameterPlaceholders(json, ParamUtils.convert(paramsMap)); + json = ParameterUtils.convertParameterPlaceholders(json, ParameterUtils.convert(paramsMap)); log.debug("datax job json : {}", json); @@ -412,7 +411,7 @@ public class DataxTask extends AbstractTask { jobConfigFilePath; // replace placeholder - String dataxCommand = ParameterUtils.convertParameterPlaceholders(sbr, ParamUtils.convert(paramsMap)); + String dataxCommand = ParameterUtils.convertParameterPlaceholders(sbr, ParameterUtils.convert(paramsMap)); log.debug("raw script : {}", dataxCommand); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTask.java index d4d478bba1..74cf5b9b29 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTask.java @@ -27,8 +27,7 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractRemoteTask; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.commons.beanutils.BeanUtils; import org.apache.commons.lang3.StringUtils; @@ -236,7 +235,7 @@ public class DmsTask extends AbstractRemoteTask { // create a new parameter object using the json data if the json data is not empty if (parameters.getIsJsonFormat() && parameters.getJsonData() != null) { String jsonData = ParameterUtils.convertParameterPlaceholders(parameters.getJsonData(), - ParamUtils.convert(taskExecutionContext.getPrepareParamsMap())); + ParameterUtils.convert(taskExecutionContext.getPrepareParamsMap())); boolean isRestartTask = parameters.getIsRestartTask(); try { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTask.java index 4da841b0d3..4fc391c839 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTask.java @@ -22,8 +22,8 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.api.stream.StreamTask; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.commons.collections4.CollectionUtils; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtils.java index f56a69e553..a96cddfd08 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtils.java @@ -20,9 +20,8 @@ package org.apache.dolphinscheduler.plugin.task.flink; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ArgsUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.commons.lang3.StringUtils; @@ -293,7 +292,7 @@ public class FlinkArgsUtils { String mainArgs = flinkParameters.getMainArgs(); if (StringUtils.isNotEmpty(mainArgs)) { Map paramsMap = taskExecutionContext.getPrepareParamsMap(); - args.add(ParameterUtils.convertParameterPlaceholders(mainArgs, ParamUtils.convert(paramsMap))); + args.add(ParameterUtils.convertParameterPlaceholders(mainArgs, ParameterUtils.convert(paramsMap))); } // determine yarn queue diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTask.java index 01862816b6..abf0da2c5c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTask.java @@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractYarnTask; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import java.util.List; import java.util.regex.Matcher; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTask.java index f76e046c17..0aaf5f70b2 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTask.java @@ -29,8 +29,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.commons.lang3.StringUtils; @@ -130,7 +129,7 @@ public class HiveCliTask extends AbstractRemoteTask { final Map paramsMap = taskExecutionContext.getPrepareParamsMap(); final String command = - ParameterUtils.convertParameterPlaceholders(String.join(" ", args), ParamUtils.convert(paramsMap)); + ParameterUtils.convertParameterPlaceholders(String.join(" ", args), ParameterUtils.convert(paramsMap)); log.info("hiveCli task command: {}", command); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java index da8b4494e3..8d696a19b1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java @@ -29,8 +29,7 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.Charsets; @@ -140,14 +139,15 @@ public class HttpTask extends AbstractTask { if (CollectionUtils.isNotEmpty(httpParameters.getHttpParams())) { for (HttpProperty httpProperty : httpParameters.getHttpParams()) { String jsonObject = JSONUtils.toJsonString(httpProperty); - String params = ParameterUtils.convertParameterPlaceholders(jsonObject, ParamUtils.convert(paramsMap)); + String params = + ParameterUtils.convertParameterPlaceholders(jsonObject, ParameterUtils.convert(paramsMap)); log.info("http request params：{}", params); httpPropertyList.add(JSONUtils.parseObject(params, HttpProperty.class)); } } addRequestParams(builder, httpPropertyList); String requestUrl = - ParameterUtils.convertParameterPlaceholders(httpParameters.getUrl(), ParamUtils.convert(paramsMap)); + ParameterUtils.convertParameterPlaceholders(httpParameters.getUrl(), ParameterUtils.convert(paramsMap)); HttpUriRequest request = builder.setUri(requestUrl).build(); setHeaders(request, httpPropertyList); return client.execute(request); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java index d05cbec1bd..9486ba5003 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java @@ -32,9 +32,8 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.java.exception.JavaSourceFileExistException; import org.apache.dolphinscheduler.plugin.task.java.exception.PublicClassNotFoundException; import org.apache.dolphinscheduler.plugin.task.java.exception.RunTypeNotFoundException; @@ -324,7 +323,7 @@ public class JavaTask extends AbstractTask { paramsMap.putAll(taskRequest.getParamsMap()); } log.info("The current java source code will begin to replace the placeholder: {}", rawJavaScript); - rawJavaScript = ParameterUtils.convertParameterPlaceholders(rawJavaScript, ParamUtils.convert(paramsMap)); + rawJavaScript = ParameterUtils.convertParameterPlaceholders(rawJavaScript, ParameterUtils.convert(paramsMap)); return rawJavaScript; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTask.java index ea43814b5b..e3058ee329 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTask.java @@ -29,8 +29,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.commons.lang3.StringUtils; @@ -151,7 +150,7 @@ public class JupyterTask extends AbstractRemoteTask { // replace placeholder Map paramsMap = taskExecutionContext.getPrepareParamsMap(); String command = ParameterUtils - .convertParameterPlaceholders(String.join(" ", args), ParamUtils.convert(paramsMap)); + .convertParameterPlaceholders(String.join(" ", args), ParameterUtils.convert(paramsMap)); log.info("jupyter task command: {}", command); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java index d35c1999cf..35e7854bab 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java @@ -30,7 +30,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.NodeSelectorExpression; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.K8sTaskParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -91,7 +91,7 @@ public class K8sTask extends AbstractK8sTask { k8sTaskMainParameters.setClusterName(clusterName); k8sTaskMainParameters.setMinCpuCores(k8sTaskParameters.getMinCpuCores()); k8sTaskMainParameters.setMinMemorySpace(k8sTaskParameters.getMinMemorySpace()); - k8sTaskMainParameters.setParamsMap(ParamUtils.convert(paramsMap)); + k8sTaskMainParameters.setParamsMap(ParameterUtils.convert(paramsMap)); k8sTaskMainParameters.setLabelMap(convertToLabelMap(k8sTaskParameters.getCustomizedLabels())); k8sTaskMainParameters .setNodeSelectorRequirements(convertToNodeSelectorRequirements(k8sTaskParameters.getNodeSelectors())); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-kubeflow/src/main/java/org/apache/dolphinscheduler/plugin/kubeflow/KubeflowTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-kubeflow/src/main/java/org/apache/dolphinscheduler/plugin/kubeflow/KubeflowTask.java index d16c472f0f..4ae9a95a5d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-kubeflow/src/main/java/org/apache/dolphinscheduler/plugin/kubeflow/KubeflowTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-kubeflow/src/main/java/org/apache/dolphinscheduler/plugin/kubeflow/KubeflowTask.java @@ -25,8 +25,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import java.io.IOException; import java.nio.file.Files; @@ -132,7 +131,7 @@ public class KubeflowTask extends AbstractRemoteTask { String clusterYAML = kubeflowParameters.getClusterYAML(); Map paramsMap = taskExecutionContext.getPrepareParamsMap(); - yamlContent = ParameterUtils.convertParameterPlaceholders(yamlContent, ParamUtils.convert(paramsMap)); + yamlContent = ParameterUtils.convertParameterPlaceholders(yamlContent, ParameterUtils.convert(paramsMap)); yamlPath = Paths.get(taskExecutionContext.getExecutePath(), KubeflowHelper.CONSTANTS.YAML_FILE_PATH); clusterYAMLPath = diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-linkis/src/main/java/org/apache/dolphinscheduler/plugin/task/linkis/LinkisTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-linkis/src/main/java/org/apache/dolphinscheduler/plugin/task/linkis/LinkisTask.java index c75bedf117..132e6f6e56 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-linkis/src/main/java/org/apache/dolphinscheduler/plugin/task/linkis/LinkisTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-linkis/src/main/java/org/apache/dolphinscheduler/plugin/task/linkis/LinkisTask.java @@ -29,8 +29,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; @@ -251,6 +250,6 @@ public class LinkisTask extends AbstractRemoteTask { private String parseScript(String script) { Map paramsMap = taskExecutionContext.getPrepareParamsMap(); - return ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); + return ParameterUtils.convertParameterPlaceholders(script, ParameterUtils.convert(paramsMap)); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java index 642985c803..e4fe074c05 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java @@ -30,8 +30,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.commons.lang3.StringUtils; @@ -221,7 +220,7 @@ public class MlflowTask extends AbstractTask { } args.add(runCommand); - return ParameterUtils.convertParameterPlaceholders(String.join("\n", args), ParamUtils.convert(paramsMap)); + return ParameterUtils.convertParameterPlaceholders(String.join("\n", args), ParameterUtils.convert(paramsMap)); } /** @@ -249,7 +248,7 @@ public class MlflowTask extends AbstractTask { imageName)); } - return ParameterUtils.convertParameterPlaceholders(String.join("\n", args), ParamUtils.convert(paramsMap)); + return ParameterUtils.convertParameterPlaceholders(String.join("\n", args), ParameterUtils.convert(paramsMap)); } private Map getParamsMap() { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTask.java index 988711370f..e172488103 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTask.java @@ -23,8 +23,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import java.util.ArrayList; import java.util.List; @@ -77,12 +76,12 @@ public class MapReduceTask extends AbstractYarnTask { Map paramsMap = taskExecutionContext.getPrepareParamsMap(); String args = ParameterUtils.convertParameterPlaceholders(mapreduceParameters.getMainArgs(), - ParamUtils.convert(paramsMap)); + ParameterUtils.convert(paramsMap)); mapreduceParameters.setMainArgs(args); if (mapreduceParameters.getProgramType() != null && mapreduceParameters.getProgramType() == ProgramType.PYTHON) { String others = ParameterUtils.convertParameterPlaceholders(mapreduceParameters.getOthers(), - ParamUtils.convert(paramsMap)); + ParameterUtils.convert(paramsMap)); mapreduceParameters.setOthers(others); } log.info("Initialize mapreduce task params {}", JSONUtils.toPrettyJsonString(mapreduceParameters)); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTask.java index 88371675ef..19b5104f07 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTask.java @@ -21,8 +21,7 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.python.PythonTask; import org.apache.commons.lang3.StringUtils; @@ -92,7 +91,7 @@ public class OpenmldbTask extends PythonTask { String rawSQLScript = openmldbParameters.getSql().replaceAll("[\\r]?\\n", "\n"); Map paramsMap = mergeParamsWithContext(openmldbParameters); - rawSQLScript = ParameterUtils.convertParameterPlaceholders(rawSQLScript, ParamUtils.convert(paramsMap)); + rawSQLScript = ParameterUtils.convertParameterPlaceholders(rawSQLScript, ParameterUtils.convert(paramsMap)); // convert sql to python script String pythonScript = buildPythonScriptsFromSql(rawSQLScript); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java index a79d189657..cff003d26d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java @@ -32,7 +32,7 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbType; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTask.java index f2853bab5e..f60d5f1107 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTask.java @@ -27,8 +27,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.commons.io.FileUtils; @@ -172,7 +171,7 @@ public class PythonTask extends AbstractTask { log.info("raw python script : {}", pythonParameters.getRawScript()); String rawPythonScript = pythonParameters.getRawScript().replaceAll("\\r\\n", System.lineSeparator()); Map paramsMap = mergeParamsWithContext(pythonParameters); - return ParameterUtils.convertParameterPlaceholders(rawPythonScript, ParamUtils.convert(paramsMap)); + return ParameterUtils.convertParameterPlaceholders(rawPythonScript, ParameterUtils.convert(paramsMap)); } protected Map mergeParamsWithContext(AbstractParameters parameters) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTask.java index f4f6d43a68..afe371628c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTask.java @@ -27,8 +27,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import java.util.ArrayList; import java.util.List; @@ -119,7 +118,7 @@ public class PytorchTask extends AbstractTask { } Map paramsMap = taskExecutionContext.getPrepareParamsMap(); - return ParameterUtils.convertParameterPlaceholders(String.join("\n", args), ParamUtils.convert(paramsMap)); + return ParameterUtils.convertParameterPlaceholders(String.join("\n", args), ParameterUtils.convert(paramsMap)); } private String getPythonCommand() { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteShellTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteShellTask.java index c0af396499..216203bfea 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteShellTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteShellTask.java @@ -31,8 +31,7 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.commons.lang3.SystemUtils; @@ -168,7 +167,7 @@ public class RemoteShellTask extends AbstractTask { private String parseScript(String script) { Map paramsMap = taskExecutionContext.getPrepareParamsMap(); - return ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); + return ParameterUtils.convertParameterPlaceholders(script, ParameterUtils.convert(paramsMap)); } public void initRemoteExecutor() { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTask.java index b15d76516c..0e8a296151 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTask.java @@ -29,8 +29,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.commons.lang3.StringUtils; @@ -167,7 +166,7 @@ public class SagemakerTask extends AbstractRemoteTask { private String parseRequstJson(String requestJson) { Map paramsMap = taskRequest.getPrepareParamsMap(); - return ParameterUtils.convertParameterPlaceholders(requestJson, ParamUtils.convert(paramsMap)); + return ParameterUtils.convertParameterPlaceholders(requestJson, ParameterUtils.convert(paramsMap)); } protected AmazonSageMaker createClient() { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTask.java index 2e124edd9c..bbf42ed8fa 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTask.java @@ -30,8 +30,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.BooleanUtils; @@ -205,7 +204,7 @@ public class SeatunnelTask extends AbstractRemoteTask { private String parseScript(String script) { Map paramsMap = taskExecutionContext.getPrepareParamsMap(); - return ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); + return ParameterUtils.convertParameterPlaceholders(script, ParameterUtils.convert(paramsMap)); } public void setSeatunnelParameters(SeatunnelParameters seatunnelParameters) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java index 469c30bb12..7d8fdd2e5a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java @@ -29,8 +29,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.commons.lang3.SystemUtils; @@ -157,6 +156,6 @@ public class ShellTask extends AbstractTask { private String parseScript(String script) { Map paramsMap = taskExecutionContext.getPrepareParamsMap(); - return ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); + return ParameterUtils.convertParameterPlaceholders(script, ParameterUtils.convert(paramsMap)); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java index b384f9b63c..afe2c1d223 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java @@ -30,9 +30,8 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ArgsUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; @@ -118,7 +117,7 @@ public class SparkTask extends AbstractYarnTask { Map paramsMap = taskExecutionContext.getPrepareParamsMap(); String command = - ParameterUtils.convertParameterPlaceholders(String.join(" ", args), ParamUtils.convert(paramsMap)); + ParameterUtils.convertParameterPlaceholders(String.join(" ", args), ParameterUtils.convert(paramsMap)); log.info("spark task command: {}", command); @@ -269,7 +268,7 @@ public class SparkTask extends AbstractYarnTask { script = script.replaceAll("\\r\\n", System.lineSeparator()); // replace placeholder Map paramsMap = taskExecutionContext.getPrepareParamsMap(); - script = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); + script = ParameterUtils.convertParameterPlaceholders(script, ParameterUtils.convert(paramsMap)); return script; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java index f432d5b4e3..b8dfde23f3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java @@ -36,8 +36,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.TaskAlertInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.SqlParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.UdfFuncParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbType; @@ -492,7 +491,7 @@ public class SqlTask extends AbstractTask { if (StringUtils.isNotEmpty(sqlParameters.getTitle())) { String title = ParameterUtils.convertParameterPlaceholders(sqlParameters.getTitle(), - ParamUtils.convert(paramsMap)); + ParameterUtils.convert(paramsMap)); log.info("SQL title : {}", title); sqlParameters.setTitle(title); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java index 506b2b1853..b8d35e4fdf 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java @@ -24,8 +24,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.sqoop.generator.SqoopJobGenerator; import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.SqoopParameters; @@ -80,7 +79,7 @@ public class SqoopTask extends AbstractYarnTask { Map paramsMap = taskExecutionContext.getPrepareParamsMap(); - String resultScripts = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); + String resultScripts = ParameterUtils.convertParameterPlaceholders(script, ParameterUtils.convert(paramsMap)); log.info("sqoop script: {}", resultScripts); return resultScripts;