From c3809fbecfc87999991da626ec0febc6adadeae0 Mon Sep 17 00:00:00 2001 From: Aaron Wang Date: Tue, 16 May 2023 12:04:33 +0800 Subject: [PATCH] [Improvement-13868][K8S Task] Support the custom labels for k8s task (#14067) * [Improvement-13868] Support the custom labels for k8s task * fix front-end type mismatched --- docs/docs/en/guide/task/kubernetes.md | 1 + docs/docs/zh/guide/task/kubernetes.md | 1 + .../task/api/k8s/K8sTaskMainParameters.java | 1 + .../task/api/k8s/impl/K8sTaskExecutor.java | 2 +- .../plugin/task/api/model/Label.java | 40 +++++++++ .../api/parameters/K8sTaskParameters.java | 2 + .../task/api/k8s/K8sTaskExecutorTest.java | 4 + .../plugin/task/k8s/K8sTask.java | 17 ++++ .../plugin/task/k8s/K8sParametersTest.java | 7 ++ .../plugin/task/k8s/K8sTaskTest.java | 10 ++- .../src/components/form/fields/get-field.ts | 3 +- .../src/components/form/types.ts | 1 + .../src/locales/en_US/project.ts | 4 + .../src/locales/zh_CN/project.ts | 4 + .../task/components/node/fields/index.ts | 1 + .../node/fields/use-custom-labels.ts | 82 +++++++++++++++++++ .../task/components/node/fields/use-k8s.ts | 3 +- .../task/components/node/format-data.ts | 1 + .../task/components/node/tasks/use-k8s.ts | 1 + .../projects/task/components/node/types.ts | 6 ++ 20 files changed, 186 insertions(+), 5 deletions(-) create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/Label.java create mode 100644 dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-custom-labels.ts diff --git a/docs/docs/en/guide/task/kubernetes.md b/docs/docs/en/guide/task/kubernetes.md index e106f42511..aa0660c423 100644 --- a/docs/docs/en/guide/task/kubernetes.md +++ b/docs/docs/en/guide/task/kubernetes.md @@ -24,6 +24,7 @@ K8S task type used to execute a batch task. In this task, the worker submits the | Image | The registry url for image. | | Command | The container execution command (yaml-style array), for example: ["printenv"] | | Args | The args of execution command (yaml-style array), for example: ["HOSTNAME", "KUBERNETES_PORT"] | +| Custom label | The customized labels for k8s Job. | | Custom parameter | It is a local user-defined parameter for K8S task, these params will pass to container as environment variables. | ## Task Example diff --git a/docs/docs/zh/guide/task/kubernetes.md b/docs/docs/zh/guide/task/kubernetes.md index 73a3c5c67c..bc5959c093 100644 --- a/docs/docs/zh/guide/task/kubernetes.md +++ b/docs/docs/zh/guide/task/kubernetes.md @@ -24,6 +24,7 @@ kubernetes任务类型,用于在kubernetes上执行一个短时和批处理的 | 镜像 | 镜像地址 | | 容器执行命令 | 容器执行命令(yaml格式数组),例如:["printenv"] | | 执行命令参数 | 执行命令参数(yaml格式数组),例如:["HOSTNAME", "KUBERNETES_PORT"] | +| 自定义标签 | 作业自定义标签 | | 自定义参数 | kubernetes任务局部的用户自定义参数,自定义参数最终会通过环境变量形式存在于容器中,提供给kubernetes任务使用 | ## 任务样例 diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java index 55f66fb9a4..43d7e4278f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java @@ -35,4 +35,5 @@ public class K8sTaskMainParameters { private double minCpuCores; private double minMemorySpace; private Map paramsMap; + private Map labelMap; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java index 50f09fa7c0..ea15cdb5f8 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java @@ -94,7 +94,7 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor { Map limitRes = new HashMap<>(); limitRes.put(MEMORY, new Quantity(String.format("%s%s", limitPodMem, MI))); limitRes.put(CPU, new Quantity(String.valueOf(limitPodCpu))); - Map labelMap = new HashMap<>(); + Map labelMap = k8STaskMainParameters.getLabelMap(); labelMap.put(LAYER_LABEL, LAYER_LABEL_VALUE); labelMap.put(NAME_LABEL, k8sJobName); EnvVar taskInstanceIdVar = new EnvVar(TASK_INSTANCE_ID, taskInstanceId, null); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/Label.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/Label.java new file mode 100644 index 0000000000..b56310d02e --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/Label.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.api.model; + +import java.io.Serializable; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class Label implements Serializable { + + /** + * label name + */ + private String label; + + /** + * label value + */ + private String value; +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java index db7a368333..d11cc381ab 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters; +import org.apache.dolphinscheduler.plugin.task.api.model.Label; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.commons.lang3.StringUtils; @@ -35,6 +36,7 @@ public class K8sTaskParameters extends AbstractParameters { private String image; private String namespace; private String command; + private List