From 1011852ab25c7536a0a64448be310bb9c3619d25 Mon Sep 17 00:00:00 2001 From: Rick Cheng Date: Fri, 26 May 2023 09:29:09 +0800 Subject: [PATCH] [Improvement-14195][Parameter] remove misleading comment of combining local and global parameter (#14196) --- .../org/apache/dolphinscheduler/plugin/task/dms/DmsTask.java | 1 - .../dolphinscheduler/plugin/task/jupyter/JupyterTask.java | 2 +- .../dolphinscheduler/plugin/task/linkis/LinkisTask.java | 1 - .../dolphinscheduler/plugin/task/mlflow/MlflowTask.java | 1 - .../dolphinscheduler/plugin/task/procedure/ProcedureTask.java | 1 - .../plugin/task/remoteshell/RemoteShellTask.java | 1 - .../dolphinscheduler/plugin/task/sagemaker/SagemakerTask.java | 1 - .../dolphinscheduler/plugin/task/seatunnel/SeatunnelTask.java | 1 - .../apache/dolphinscheduler/plugin/task/shell/ShellTask.java | 1 - .../apache/dolphinscheduler/plugin/task/spark/SparkTask.java | 4 ++-- .../org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java | 2 +- .../apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java | 1 - 12 files changed, 4 insertions(+), 13 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTask.java index 5b29d54a46..d4d478bba1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTask.java @@ -235,7 +235,6 @@ public class DmsTask extends AbstractRemoteTask { public void convertJsonParameters() throws TaskException { // create a new parameter object using the json data if the json data is not empty if (parameters.getIsJsonFormat() && parameters.getJsonData() != null) { - // combining local and global parameters String jsonData = ParameterUtils.convertParameterPlaceholders(parameters.getJsonData(), ParamUtils.convert(taskExecutionContext.getPrepareParamsMap())); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTask.java index f8d13b77b3..ea43814b5b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTask.java @@ -148,7 +148,7 @@ public class JupyterTask extends AbstractRemoteTask { args.add(String.format(JupyterConstants.REMOVE_ENV, timestamp)); } - // replace placeholder, and combining local and global parameters + // replace placeholder Map paramsMap = taskExecutionContext.getPrepareParamsMap(); String command = ParameterUtils .convertParameterPlaceholders(String.join(" ", args), ParamUtils.convert(paramsMap)); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-linkis/src/main/java/org/apache/dolphinscheduler/plugin/task/linkis/LinkisTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-linkis/src/main/java/org/apache/dolphinscheduler/plugin/task/linkis/LinkisTask.java index acb54c9ebd..c75bedf117 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-linkis/src/main/java/org/apache/dolphinscheduler/plugin/task/linkis/LinkisTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-linkis/src/main/java/org/apache/dolphinscheduler/plugin/task/linkis/LinkisTask.java @@ -250,7 +250,6 @@ public class LinkisTask extends AbstractRemoteTask { } private String parseScript(String script) { - // combining local and global parameters Map paramsMap = taskExecutionContext.getPrepareParamsMap(); return ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java index cd2487920a..b907a22eba 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java @@ -253,7 +253,6 @@ public class MlflowTask extends AbstractTask { } private Map getParamsMap() { - // replace placeholder, and combining local and global parameters return taskExecutionContext.getPrepareParamsMap(); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java index c0fecc2513..a79d189657 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java @@ -137,7 +137,6 @@ public class ProcedureTask extends AbstractTask { } private String formatSql(Map sqlParamsMap, Map paramsMap) { - // combining local and global parameters setSqlParamsMap(procedureParameters.getMethod(), rgex, sqlParamsMap, paramsMap, taskExecutionContext.getTaskInstanceId()); return procedureParameters.getMethod().replaceAll(rgex, "?"); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteShellTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteShellTask.java index 48644d7f3c..c6eb8084e6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteShellTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteShellTask.java @@ -167,7 +167,6 @@ public class RemoteShellTask extends AbstractTask { } private String parseScript(String script) { - // combining local and global parameters Map paramsMap = taskExecutionContext.getPrepareParamsMap(); return ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTask.java index 76109082f8..b15d76516c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTask.java @@ -166,7 +166,6 @@ public class SagemakerTask extends AbstractRemoteTask { } private String parseRequstJson(String requestJson) { - // combining local and global parameters Map paramsMap = taskRequest.getPrepareParamsMap(); return ParameterUtils.convertParameterPlaceholders(requestJson, ParamUtils.convert(paramsMap)); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTask.java index 836df8b6c6..2e124edd9c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTask.java @@ -204,7 +204,6 @@ public class SeatunnelTask extends AbstractRemoteTask { } private String parseScript(String script) { - // combining local and global parameters Map paramsMap = taskExecutionContext.getPrepareParamsMap(); return ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java index 5c1ea5991d..dfae160da6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java @@ -156,7 +156,6 @@ public class ShellTask extends AbstractTask { } private String parseScript(String script) { - // combining local and global parameters Map paramsMap = taskExecutionContext.getPrepareParamsMap(); return ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java index 00a12430a3..b384f9b63c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java @@ -114,7 +114,7 @@ public class SparkTask extends AbstractYarnTask { // populate spark options args.addAll(populateSparkOptions()); - // replace placeholder, and combining local and global parameters + // replace placeholder Map paramsMap = taskExecutionContext.getPrepareParamsMap(); String command = @@ -267,7 +267,7 @@ public class SparkTask extends AbstractYarnTask { private String replaceParam(String script) { script = script.replaceAll("\\r\\n", System.lineSeparator()); - // replace placeholder, and combining local and global parameters + // replace placeholder Map paramsMap = taskExecutionContext.getPrepareParamsMap(); script = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); return script; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java index a0935f0266..f432d5b4e3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java @@ -481,7 +481,7 @@ public class SqlTask extends AbstractTask { // replace variable TIME with $[YYYYmmddd...] in sql when history run job and batch complement job sql = ParameterUtils.replaceScheduleTime(sql, DateUtils.timeStampToDate(taskExecutionContext.getScheduleTime())); - // combining local and global parameters + Map paramsMap = taskExecutionContext.getPrepareParamsMap(); // spell SQL according to the final user-defined variable diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java index d5a5d87c8b..506b2b1853 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java @@ -78,7 +78,6 @@ public class SqoopTask extends AbstractYarnTask { SqoopJobGenerator generator = new SqoopJobGenerator(); String script = generator.generateSqoopJob(sqoopParameters, sqoopTaskExecutionContext); - // combining local and global parameters Map paramsMap = taskExecutionContext.getPrepareParamsMap(); String resultScripts = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));