Browse Source

[Improvement-14195][Parameter] remove misleading comment of combining local and global parameter (#14196)

3.2.0-release
Rick Cheng 2 years ago committed by GitHub
parent
commit
1011852ab2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTask.java
  2. 2
      dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTask.java
  3. 1
      dolphinscheduler-task-plugin/dolphinscheduler-task-linkis/src/main/java/org/apache/dolphinscheduler/plugin/task/linkis/LinkisTask.java
  4. 1
      dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java
  5. 1
      dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java
  6. 1
      dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteShellTask.java
  7. 1
      dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTask.java
  8. 1
      dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTask.java
  9. 1
      dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java
  10. 4
      dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java
  11. 2
      dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java
  12. 1
      dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java

1
dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTask.java

@ -235,7 +235,6 @@ public class DmsTask extends AbstractRemoteTask {
public void convertJsonParameters() throws TaskException { public void convertJsonParameters() throws TaskException {
// create a new parameter object using the json data if the json data is not empty // create a new parameter object using the json data if the json data is not empty
if (parameters.getIsJsonFormat() && parameters.getJsonData() != null) { if (parameters.getIsJsonFormat() && parameters.getJsonData() != null) {
// combining local and global parameters
String jsonData = ParameterUtils.convertParameterPlaceholders(parameters.getJsonData(), String jsonData = ParameterUtils.convertParameterPlaceholders(parameters.getJsonData(),
ParamUtils.convert(taskExecutionContext.getPrepareParamsMap())); ParamUtils.convert(taskExecutionContext.getPrepareParamsMap()));

2
dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTask.java

@ -148,7 +148,7 @@ public class JupyterTask extends AbstractRemoteTask {
args.add(String.format(JupyterConstants.REMOVE_ENV, timestamp)); args.add(String.format(JupyterConstants.REMOVE_ENV, timestamp));
} }
// replace placeholder, and combining local and global parameters // replace placeholder
Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap(); Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap();
String command = ParameterUtils String command = ParameterUtils
.convertParameterPlaceholders(String.join(" ", args), ParamUtils.convert(paramsMap)); .convertParameterPlaceholders(String.join(" ", args), ParamUtils.convert(paramsMap));

1
dolphinscheduler-task-plugin/dolphinscheduler-task-linkis/src/main/java/org/apache/dolphinscheduler/plugin/task/linkis/LinkisTask.java

@ -250,7 +250,6 @@ public class LinkisTask extends AbstractRemoteTask {
} }
private String parseScript(String script) { private String parseScript(String script) {
// combining local and global parameters
Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap(); Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap();
return ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); return ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
} }

1
dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java

@ -253,7 +253,6 @@ public class MlflowTask extends AbstractTask {
} }
private Map<String, Property> getParamsMap() { private Map<String, Property> getParamsMap() {
// replace placeholder, and combining local and global parameters
return taskExecutionContext.getPrepareParamsMap(); return taskExecutionContext.getPrepareParamsMap();
} }

1
dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java

@ -137,7 +137,6 @@ public class ProcedureTask extends AbstractTask {
} }
private String formatSql(Map<Integer, Property> sqlParamsMap, Map<String, Property> paramsMap) { private String formatSql(Map<Integer, Property> sqlParamsMap, Map<String, Property> paramsMap) {
// combining local and global parameters
setSqlParamsMap(procedureParameters.getMethod(), rgex, sqlParamsMap, paramsMap, setSqlParamsMap(procedureParameters.getMethod(), rgex, sqlParamsMap, paramsMap,
taskExecutionContext.getTaskInstanceId()); taskExecutionContext.getTaskInstanceId());
return procedureParameters.getMethod().replaceAll(rgex, "?"); return procedureParameters.getMethod().replaceAll(rgex, "?");

1
dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteShellTask.java

@ -167,7 +167,6 @@ public class RemoteShellTask extends AbstractTask {
} }
private String parseScript(String script) { private String parseScript(String script) {
// combining local and global parameters
Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap(); Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap();
return ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); return ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
} }

1
dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTask.java

@ -166,7 +166,6 @@ public class SagemakerTask extends AbstractRemoteTask {
} }
private String parseRequstJson(String requestJson) { private String parseRequstJson(String requestJson) {
// combining local and global parameters
Map<String, Property> paramsMap = taskRequest.getPrepareParamsMap(); Map<String, Property> paramsMap = taskRequest.getPrepareParamsMap();
return ParameterUtils.convertParameterPlaceholders(requestJson, ParamUtils.convert(paramsMap)); return ParameterUtils.convertParameterPlaceholders(requestJson, ParamUtils.convert(paramsMap));
} }

1
dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTask.java

@ -204,7 +204,6 @@ public class SeatunnelTask extends AbstractRemoteTask {
} }
private String parseScript(String script) { private String parseScript(String script) {
// combining local and global parameters
Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap(); Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap();
return ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); return ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
} }

1
dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java

@ -156,7 +156,6 @@ public class ShellTask extends AbstractTask {
} }
private String parseScript(String script) { private String parseScript(String script) {
// combining local and global parameters
Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap(); Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap();
return ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); return ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
} }

4
dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java

@ -114,7 +114,7 @@ public class SparkTask extends AbstractYarnTask {
// populate spark options // populate spark options
args.addAll(populateSparkOptions()); args.addAll(populateSparkOptions());
// replace placeholder, and combining local and global parameters // replace placeholder
Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap(); Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap();
String command = String command =
@ -267,7 +267,7 @@ public class SparkTask extends AbstractYarnTask {
private String replaceParam(String script) { private String replaceParam(String script) {
script = script.replaceAll("\\r\\n", System.lineSeparator()); script = script.replaceAll("\\r\\n", System.lineSeparator());
// replace placeholder, and combining local and global parameters // replace placeholder
Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap(); Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap();
script = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); script = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
return script; return script;

2
dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java

@ -481,7 +481,7 @@ public class SqlTask extends AbstractTask {
// replace variable TIME with $[YYYYmmddd...] in sql when history run job and batch complement job // replace variable TIME with $[YYYYmmddd...] in sql when history run job and batch complement job
sql = ParameterUtils.replaceScheduleTime(sql, sql = ParameterUtils.replaceScheduleTime(sql,
DateUtils.timeStampToDate(taskExecutionContext.getScheduleTime())); DateUtils.timeStampToDate(taskExecutionContext.getScheduleTime()));
// combining local and global parameters
Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap(); Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap();
// spell SQL according to the final user-defined variable // spell SQL according to the final user-defined variable

1
dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java

@ -78,7 +78,6 @@ public class SqoopTask extends AbstractYarnTask {
SqoopJobGenerator generator = new SqoopJobGenerator(); SqoopJobGenerator generator = new SqoopJobGenerator();
String script = generator.generateSqoopJob(sqoopParameters, sqoopTaskExecutionContext); String script = generator.generateSqoopJob(sqoopParameters, sqoopTaskExecutionContext);
// combining local and global parameters
Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap(); Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap();
String resultScripts = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); String resultScripts = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));

Loading…
Cancel
Save