From 24ff70dc88dd56112100bb4c8ebad3cfd0da2e18 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Fri, 23 Feb 2024 11:11:06 +0800 Subject: [PATCH] Fix out parameter may loss (#15617) --- .../api/parameters/AbstractParameters.java | 47 ++++------- .../plugin/task/shell/ShellParameters.java | 27 ++----- .../task/shell/ShellParametersTest.java | 77 +++++++++++++++++++ 3 files changed, 96 insertions(+), 55 deletions(-) create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/test/java/org/apache/dolphinscheduler/plugin/task/shell/ShellParametersTest.java diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java index 6ca1be7d7a..f11a83bc54 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java @@ -36,14 +36,22 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; +@Getter @Slf4j public abstract class AbstractParameters implements IParameters { + @Setter + public List localParams; + + public List varPool = new ArrayList<>(); + @Override public abstract boolean checkParameters(); @@ -52,33 +60,6 @@ public abstract class AbstractParameters implements IParameters { return new ArrayList<>(); } - /** - * local parameters - */ - public List localParams; - - /** - * var pool - */ - public List varPool; - - /** - * get local parameters list - * - * @return Property list - */ - public List getLocalParams() { - return localParams; - } - - public void setLocalParams(List localParams) { - this.localParams = localParams; - } - - /** - * get local parameters map - * @return parameters map - */ public Map getLocalParametersMap() { Map localParametersMaps = new LinkedHashMap<>(); if (localParams != null) { @@ -131,10 +112,6 @@ public abstract class AbstractParameters implements IParameters { return varPoolMap; } - public List getVarPool() { - return varPool; - } - public void setVarPool(String varPool) { if (StringUtils.isEmpty(varPool)) { this.varPool = new ArrayList<>(); @@ -161,8 +138,12 @@ public abstract class AbstractParameters implements IParameters { if (StringUtils.isNotEmpty(propValue)) { info.setValue(propValue); addPropertyToValPool(info); - } else { - log.warn("Cannot find the output parameter {} in the task output parameters", info.getProp()); + continue; + } + addPropertyToValPool(info); + if (StringUtils.isEmpty(info.getValue())) { + log.warn("The output parameter {} value is empty and cannot find the out parameter from task output", + info); } } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellParameters.java index ffbaf8cc90..42ad537f95 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellParameters.java @@ -22,34 +22,17 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters import java.util.List; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter public class ShellParameters extends AbstractParameters { - /** - * shell script - */ private String rawScript; - /** - * resource list - */ private List resourceList; - public String getRawScript() { - return rawScript; - } - - public void setRawScript(String rawScript) { - this.rawScript = rawScript; - } - - public List getResourceList() { - return resourceList; - } - - public void setResourceList(List resourceList) { - this.resourceList = resourceList; - } - @Override public boolean checkParameters() { return rawScript != null && !rawScript.isEmpty(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/test/java/org/apache/dolphinscheduler/plugin/task/shell/ShellParametersTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/test/java/org/apache/dolphinscheduler/plugin/task/shell/ShellParametersTest.java new file mode 100644 index 0000000000..a64449bd0c --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/test/java/org/apache/dolphinscheduler/plugin/task/shell/ShellParametersTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.shell; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; +import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import com.google.common.collect.Lists; + +class ShellParametersTest { + + @Test + void dealOutParamTest() { + ShellParameters shellParameters = new ShellParameters(); + List localParams = Lists.newArrayList(new Property("a", Direct.OUT, DataType.VARCHAR, "a")); + shellParameters.setLocalParams(localParams); + + Map taskOutputParams = new HashMap<>(); + taskOutputParams.put("b", "b"); + shellParameters.dealOutParam(taskOutputParams); + List varPool = shellParameters.getVarPool(); + assertEquals(1, varPool.size()); + assertEquals("a", varPool.get(0).getValue()); + } + + @Test + void dealOutParamTest_notTaskOutput() { + ShellParameters shellParameters = new ShellParameters(); + List localParams = Lists.newArrayList(new Property("a", Direct.OUT, DataType.VARCHAR, "a")); + shellParameters.setLocalParams(localParams); + + Map taskOutputParams = new HashMap<>(); + shellParameters.dealOutParam(taskOutputParams); + List varPool = shellParameters.getVarPool(); + assertEquals(1, varPool.size()); + assertEquals("a", varPool.get(0).getValue()); + } + + @Test + void dealOutParamTest_taskOutputOverrideOut() { + ShellParameters shellParameters = new ShellParameters(); + List localParams = Lists.newArrayList(new Property("a", Direct.OUT, DataType.VARCHAR, "a")); + shellParameters.setLocalParams(localParams); + + Map taskOutputParams = new HashMap<>(); + taskOutputParams.put("a", "b"); + shellParameters.dealOutParam(taskOutputParams); + List varPool = shellParameters.getVarPool(); + assertEquals(1, varPool.size()); + assertEquals("b", varPool.get(0).getValue()); + } + +}