From 3bad68a942b5c73a5e9028a11525fa2ff705949f Mon Sep 17 00:00:00 2001 From: Aaron Wang Date: Mon, 1 May 2023 11:45:03 +0800 Subject: [PATCH] [Bug-13929][K8S Task] Set command contains single quote is invalid (#13930) * [Fix-13929][K8S Task] Set command contains single quote is invalid * fix UT --- docs/docs/en/guide/task/kubernetes.md | 3 ++- docs/docs/zh/guide/task/kubernetes.md | 3 ++- .../task/api/k8s/AbstractK8sTaskExecutor.java | 3 +++ .../task/api/k8s/K8sTaskMainParameters.java | 1 + .../task/api/k8s/impl/K8sTaskExecutor.java | 17 +++++++++++------ .../task/api/parameters/K8sTaskParameters.java | 1 + .../task/api/k8s/K8sTaskExecutorTest.java | 2 +- .../plugin/task/k8s/K8sTask.java | 1 + .../plugin/task/k8s/K8sParametersTest.java | 5 ++++- .../plugin/task/k8s/K8sTaskTest.java | 8 +++++--- .../src/locales/en_US/project.ts | 4 +++- .../src/locales/zh_CN/project.ts | 4 +++- .../task/components/node/fields/use-k8s.ts | 8 ++++++++ .../task/components/node/format-data.ts | 1 + .../projects/task/components/node/types.ts | 1 + 15 files changed, 47 insertions(+), 15 deletions(-) diff --git a/docs/docs/en/guide/task/kubernetes.md b/docs/docs/en/guide/task/kubernetes.md index 727bb6518a..e106f42511 100644 --- a/docs/docs/en/guide/task/kubernetes.md +++ b/docs/docs/en/guide/task/kubernetes.md @@ -22,7 +22,8 @@ K8S task type used to execute a batch task. In this task, the worker submits the | Min CPU | Minimum CPU requirement for running k8s task. | | Min Memory | Minimum memory requirement for running k8s task. | | Image | The registry url for image. | -| Command | The container execution command, for example: /bin/echo hello world | +| Command | The container execution command (yaml-style array), for example: ["printenv"] | +| Args | The args of execution command (yaml-style array), for example: ["HOSTNAME", "KUBERNETES_PORT"] | | Custom parameter | It is a local user-defined parameter for K8S task, these params will pass to container as environment variables. | ## Task Example diff --git a/docs/docs/zh/guide/task/kubernetes.md b/docs/docs/zh/guide/task/kubernetes.md index 39bfb4f364..73a3c5c67c 100644 --- a/docs/docs/zh/guide/task/kubernetes.md +++ b/docs/docs/zh/guide/task/kubernetes.md @@ -22,7 +22,8 @@ kubernetes任务类型，用于在kubernetes上执行一个短时和批处理的 | 最小CPU | 任务在kubernetes上运行所需的最小CPU | | 最小内存 | 任务在kubernetes上运行所需的最小内存 | | 镜像 | 镜像地址 | -| 容器执行命令 | 容器执行命令，例如：/bin/echo hello world | +| 容器执行命令 | 容器执行命令（yaml格式数组），例如：["printenv"] | +| 执行命令参数 | 执行命令参数（yaml格式数组），例如：["HOSTNAME", "KUBERNETES_PORT"] | | 自定义参数 | kubernetes任务局部的用户自定义参数，自定义参数最终会通过环境变量形式存在于容器中，提供给kubernetes任务使用 | ## 任务样例 diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTaskExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTaskExecutor.java index c7bc47db0a..6bdf8d39ed 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTaskExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTaskExecutor.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.utils.K8sUtils; import org.slf4j.Logger; +import org.yaml.snakeyaml.Yaml; public abstract class AbstractK8sTaskExecutor { @@ -32,12 +33,14 @@ public abstract class AbstractK8sTaskExecutor { protected TaskExecutionContext taskRequest; protected K8sUtils k8sUtils; protected StringBuilder logStringBuffer; + protected Yaml yaml; protected AbstractK8sTaskExecutor(Logger log, TaskExecutionContext taskRequest) { this.log = log; this.taskRequest = taskRequest; this.k8sUtils = new K8sUtils(); this.logStringBuffer = new StringBuilder(); + this.yaml = new Yaml(); } public abstract TaskResponse run(String k8sParameterStr) throws Exception; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java index c035daf013..55f66fb9a4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java @@ -29,6 +29,7 @@ public class K8sTaskMainParameters { private String image; private String command; + private String args; private String namespaceName; private String clusterName; private double minCpuCores; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java index 5b82f83f94..50f09fa7c0 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.plugin.task.api.k8s.impl; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.API_VERSION; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.COMMAND_SPLIT_REGEX; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.CPU; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_KILL; @@ -54,7 +53,6 @@ import java.util.Locale; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; import org.slf4j.Logger; @@ -112,13 +110,19 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor { } String commandString = k8STaskMainParameters.getCommand(); + String argsString = k8STaskMainParameters.getArgs(); List commands = new ArrayList<>(); + List args = new ArrayList<>(); - if (commandString != null) { - Matcher commandMatcher = COMMAND_SPLIT_REGEX.matcher(commandString.trim()); - while (commandMatcher.find()) { - commands.add(commandMatcher.group()); + try { + if (!StringUtils.isEmpty(commandString)) { + commands = yaml.load(commandString.trim()); + } + if (!StringUtils.isEmpty(argsString)) { + args = yaml.load(argsString.trim()); } + } catch (Exception e) { + throw new TaskException("Parse yaml-like commands and args failed", e); } return new JobBuilder() @@ -136,6 +140,7 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor { .withName(k8sJobName) .withImage(image) .withCommand(commands.size() == 0 ? null : commands) + .withArgs(args.size() == 0 ? null : args) .withImagePullPolicy(IMAGE_PULL_POLICY) .withResources(new ResourceRequirements(limitRes, reqRes)) .withEnv(envVars) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java index e9a8e96520..db7a368333 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java @@ -35,6 +35,7 @@ public class K8sTaskParameters extends AbstractParameters { private String image; private String namespace; private String command; + private String args; private double minCpuCores; private double minMemorySpace; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java index a4f51abdf1..c19fe8e40f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java @@ -62,7 +62,7 @@ public class K8sTaskExecutorTest { k8sTaskMainParameters.setClusterName(clusterName); k8sTaskMainParameters.setMinCpuCores(minCpuCores); k8sTaskMainParameters.setMinMemorySpace(minMemorySpace); - k8sTaskMainParameters.setCommand("echo 'hello world'"); + k8sTaskMainParameters.setCommand("[\"perl\" ,\"-Mbignum=bpi\", \"-wle\", \"print bpi(2000)\"]"); job = k8sTaskExecutor.buildK8sJob(k8sTaskMainParameters); } @Test diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java index 6a180d7d05..3f37fb396a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java @@ -83,6 +83,7 @@ public class K8sTask extends AbstractK8sTask { k8sTaskMainParameters.setMinMemorySpace(k8sTaskParameters.getMinMemorySpace()); k8sTaskMainParameters.setParamsMap(ParamUtils.convert(paramsMap)); k8sTaskMainParameters.setCommand(k8sTaskParameters.getCommand()); + k8sTaskMainParameters.setArgs(k8sTaskParameters.getArgs()); return JSONUtils.toJsonString(k8sTaskMainParameters); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sParametersTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sParametersTest.java index 0273bb1651..5f66efe2a7 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sParametersTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sParametersTest.java @@ -30,7 +30,8 @@ public class K8sParametersTest { private final String namespace = "{\"name\":\"default\",\"cluster\":\"lab\"}"; private final double minCpuCores = 2; private final double minMemorySpace = 10; - private final String command = "echo 'hello world'"; + private final String command = "[\"/bin/bash\", \"-c\"]"; + private final String args = "[\"echo hello world\"]"; @BeforeEach public void before() { @@ -40,6 +41,7 @@ public class K8sParametersTest { k8sTaskParameters.setMinCpuCores(minCpuCores); k8sTaskParameters.setMinMemorySpace(minMemorySpace); k8sTaskParameters.setCommand(command); + k8sTaskParameters.setArgs(args); } @Test @@ -60,6 +62,7 @@ public class K8sParametersTest { Assertions.assertEquals(0, Double.compare(minCpuCores, k8sTaskParameters.getMinCpuCores())); Assertions.assertEquals(0, Double.compare(minMemorySpace, k8sTaskParameters.getMinMemorySpace())); Assertions.assertEquals(command, k8sTaskParameters.getCommand()); + Assertions.assertEquals(args, k8sTaskParameters.getArgs()); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskTest.java index 79b8f05e70..00710cda23 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskTest.java @@ -48,7 +48,8 @@ public class K8sTaskTest { private final String DAY = "day"; private final String date = "20220507"; - private final String command = "echo 'hello world'"; + private final String command = "[\"/bin/bash\", \"-c\"]"; + private final String args = "[\"echo hello world\"]"; @BeforeEach public void before() { k8sTaskParameters = new K8sTaskParameters(); @@ -57,6 +58,7 @@ public class K8sTaskTest { k8sTaskParameters.setMinCpuCores(minCpuCores); k8sTaskParameters.setMinMemorySpace(minMemorySpace); k8sTaskParameters.setCommand(command); + k8sTaskParameters.setArgs(args); TaskExecutionContext taskRequest = new TaskExecutionContext(); taskRequest.setTaskInstanceId(taskInstanceId); taskRequest.setTaskName(taskName); @@ -82,7 +84,7 @@ public class K8sTaskTest { @Test public void testBuildCommandNormal() { String expectedStr = - "{\"image\":\"ds-dev\",\"command\":\"echo 'hello world'\",\"namespaceName\":\"default\",\"clusterName\":\"lab\",\"minCpuCores\":2.0,\"minMemorySpace\":10.0,\"paramsMap\":{\"day\":\"20220507\"}}"; + "{\"image\":\"ds-dev\",\"command\":\"[\\\"/bin/bash\\\", \\\"-c\\\"]\",\"args\":\"[\\\"echo hello world\\\"]\",\"namespaceName\":\"default\",\"clusterName\":\"lab\",\"minCpuCores\":2.0,\"minMemorySpace\":10.0,\"paramsMap\":{\"day\":\"20220507\"}}"; String commandStr = k8sTask.buildCommand(); Assertions.assertEquals(expectedStr, commandStr); } @@ -90,7 +92,7 @@ public class K8sTaskTest { @Test public void testGetParametersNormal() { String expectedStr = - "K8sTaskParameters(image=ds-dev, namespace={\"name\":\"default\",\"cluster\":\"lab\"}, command=echo 'hello world', minCpuCores=2.0, minMemorySpace=10.0)"; + "K8sTaskParameters(image=ds-dev, namespace={\"name\":\"default\",\"cluster\":\"lab\"}, command=[\"/bin/bash\", \"-c\"], args=[\"echo hello world\"], minCpuCores=2.0, minMemorySpace=10.0)"; String result = k8sTask.getParameters().toString(); Assertions.assertEquals(expectedStr, result); } diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts index 58ab0618da..24b90a7af3 100644 --- a/dolphinscheduler-ui/src/locales/en_US/project.ts +++ b/dolphinscheduler-ui/src/locales/en_US/project.ts @@ -379,7 +379,9 @@ export default { image_tips: 'Please enter image', command: 'Command', command_tips: - 'Please enter the container execution command, for example: /bin/echo hello world', + 'Please enter the container execution command, for example: ["printenv"]', + args: 'Args', + args_tips: 'Please enter the container execution command args, for example: ["HOSTNAME", "KUBERNETES_PORT"]', min_memory_tips: 'Please enter min memory', state: 'State', branch_flow: 'Branch flow', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts index e44c7dfa16..71086b1ce9 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts @@ -377,7 +377,9 @@ export default { image: '镜像', image_tips: '请输入镜像', command: '容器执行命令', - command_tips: '请输入容器执行命令，例如：/bin/echo hello world', + command_tips: '请输入容器执行命令，例如：["printenv"]', + args: '执行命令参数', + args_tips: '请输入容器执行命令参数，例如：["HOSTNAME", "KUBERNETES_PORT"]', min_memory_tips: '请输入最小内存', state: '状态', branch_flow: '分支流转', diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-k8s.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-k8s.ts index b6e6cbfa26..9381e397f8 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-k8s.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-k8s.ts @@ -67,6 +67,14 @@ export function useK8s(model: { [field: string]: any }): IJsonItem[] { placeholder: t('project.node.command_tips') } }, + { + type: 'input', + field: 'args', + name: t('project.node.args'), + props: { + placeholder: t('project.node.args_tips') + } + }, ...useCustomParams({ model, field: 'localParams', isSimple: true }) ] } diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts index 09f6114c5e..b3da7260d4 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts @@ -346,6 +346,7 @@ export function formatParams(data: INodeData): { taskParams.minMemorySpace = data.minMemorySpace taskParams.image = data.image taskParams.command = data.command + taskParams.args = data.args } if (data.taskType === 'JUPYTER') { diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts index 42c70535b1..c25341d86d 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts @@ -354,6 +354,7 @@ interface ITaskParams { minMemorySpace?: string image?: string command?: string + args?: string algorithm?: string params?: string searchParams?: string