From 359cbe2a1ea247a8a567762100eb80be51679c75 Mon Sep 17 00:00:00 2001 From: zixi0825 Date: Sun, 15 May 2022 17:46:31 +0800 Subject: [PATCH] [dataquality] Fix task commnd null bug (#9974) --- .../plugin/task/dq/DataQualityTask.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTask.java index d0575573a1..039987da0e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTask.java @@ -42,6 +42,7 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ArgsUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; import org.apache.dolphinscheduler.plugin.task.dq.rule.RuleManager; import org.apache.dolphinscheduler.plugin.task.dq.rule.parameter.DataQualityConfiguration; import org.apache.dolphinscheduler.plugin.task.dq.utils.spark.SparkArgsUtils; @@ -52,6 +53,7 @@ import java.io.File; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -160,8 +162,6 @@ public class DataQualityTask extends AbstractYarnTask { List args = new ArrayList<>(); args.add(SPARK2_COMMAND); - - // other parameters args.addAll(SparkArgsUtils.buildArgs(dataQualityParameters.getSparkParameters())); // replace placeholder @@ -169,10 +169,15 @@ public class DataQualityTask extends AbstractYarnTask { String command = null; - if (null != paramsMap) { - command = ParameterUtils.convertParameterPlaceholders(String.join(" ", args), ParamUtils.convert(paramsMap)); + if (MapUtils.isEmpty(paramsMap)) { + paramsMap = new HashMap<>(); + } + + if (MapUtils.isNotEmpty(dqTaskExecutionContext.getParamsMap())) { + paramsMap.putAll(dqTaskExecutionContext.getParamsMap()); } + command = ParameterUtils.convertParameterPlaceholders(String.join(" ", args), ParamUtils.convert(paramsMap)); logger.info("data quality task command: {}", command); return command; @@ -181,8 +186,8 @@ public class DataQualityTask extends AbstractYarnTask { @Override protected void setMainJarName() { ResourceInfo mainJar = new ResourceInfo(); - String basePath = System.getProperty("user.dir").replace(File.separator + "bin", File.separator + "libs"); - mainJar.setRes(basePath + File.separator + CommonUtils.getDataQualityJarName()); + String basePath = System.getProperty("user.dir").replace(File.separator + "bin", ""); + mainJar.setRes(basePath + File.separator + "libs" + File.separator + CommonUtils.getDataQualityJarName()); dataQualityParameters.getSparkParameters().setMainJar(mainJar); }