Browse Source

[Feature][K8S Task] support node selector (#14126)

* feat: support K8S node selector

* fix CI
3.2.0-release
Aaron Wang 2 years ago committed by GitHub
parent
commit
4e78248238
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 3
      docs/docs/en/guide/task/kubernetes.md
  2. 3
      docs/docs/zh/guide/task/kubernetes.md
  3. 3
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java
  4. 12
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java
  5. 45
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/NodeSelectorExpression.java
  6. 2
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java
  7. 8
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java
  8. 28
      dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java
  9. 5
      dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sParametersTest.java
  10. 8
      dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskTest.java
  11. 2
      dolphinscheduler-ui/src/components/form/fields/get-field.ts
  12. 3
      dolphinscheduler-ui/src/locales/en_US/project.ts
  13. 3
      dolphinscheduler-ui/src/locales/zh_CN/project.ts
  14. 1
      dolphinscheduler-ui/src/views/projects/task/components/node/fields/index.ts
  15. 5
      dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-k8s.ts
  16. 117
      dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-node-selectors.ts
  17. 1
      dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
  18. 1
      dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-k8s.ts
  19. 11
      dolphinscheduler-ui/src/views/projects/task/components/node/types.ts

3
docs/docs/en/guide/task/kubernetes.md

@ -17,7 +17,7 @@ K8S task type used to execute a batch task. In this task, the worker submits the
- Please refer to [DolphinScheduler Task Parameters Appendix](appendix.md) `Default Task Parameters` section for default parameters. - Please refer to [DolphinScheduler Task Parameters Appendix](appendix.md) `Default Task Parameters` section for default parameters.
| **Parameter** | **Description** | | **Parameter** | **Description** |
|------------------|------------------------------------------------------------------------------------------------------------------| |------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Namespace | The namespace for running k8s task. | | Namespace | The namespace for running k8s task. |
| Min CPU | Minimum CPU requirement for running k8s task. | | Min CPU | Minimum CPU requirement for running k8s task. |
| Min Memory | Minimum memory requirement for running k8s task. | | Min Memory | Minimum memory requirement for running k8s task. |
@ -25,6 +25,7 @@ K8S task type used to execute a batch task. In this task, the worker submits the
| Command | The container execution command (yaml-style array), for example: ["printenv"] | | Command | The container execution command (yaml-style array), for example: ["printenv"] |
| Args | The args of execution command (yaml-style array), for example: ["HOSTNAME", "KUBERNETES_PORT"] | | Args | The args of execution command (yaml-style array), for example: ["HOSTNAME", "KUBERNETES_PORT"] |
| Custom label | The customized labels for k8s Job. | | Custom label | The customized labels for k8s Job. |
| Node selector | The label selectors for running k8s pod. Different value in value set should be seperated by command, for example: `value1,value2`. You can refer to https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/node-selector-requirement/ for configuration of different operators. |
| Custom parameter | It is a local user-defined parameter for K8S task, these params will pass to container as environment variables. | | Custom parameter | It is a local user-defined parameter for K8S task, these params will pass to container as environment variables. |
## Task Example ## Task Example

3
docs/docs/zh/guide/task/kubernetes.md

@ -17,7 +17,7 @@ kubernetes任务类型,用于在kubernetes上执行一个短时和批处理的
- 默认参数说明请参考[DolphinScheduler任务参数附录](appendix.md)`默认任务参数`一栏。 - 默认参数说明请参考[DolphinScheduler任务参数附录](appendix.md)`默认任务参数`一栏。
| **任务参数** | **描述** | | **任务参数** | **描述** |
|----------|-----------------------------------------------------------------| |----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| 命名空间 | 选择kubernetes集群上存在的命名空间 | | 命名空间 | 选择kubernetes集群上存在的命名空间 |
| 最小CPU | 任务在kubernetes上运行所需的最小CPU | | 最小CPU | 任务在kubernetes上运行所需的最小CPU |
| 最小内存 | 任务在kubernetes上运行所需的最小内存 | | 最小内存 | 任务在kubernetes上运行所需的最小内存 |
@ -25,6 +25,7 @@ kubernetes任务类型,用于在kubernetes上执行一个短时和批处理的
| 容器执行命令 | 容器执行命令(yaml格式数组),例如:["printenv"] | | 容器执行命令 | 容器执行命令(yaml格式数组),例如:["printenv"] |
| 执行命令参数 | 执行命令参数(yaml格式数组),例如:["HOSTNAME", "KUBERNETES_PORT"] | | 执行命令参数 | 执行命令参数(yaml格式数组),例如:["HOSTNAME", "KUBERNETES_PORT"] |
| 自定义标签 | 作业自定义标签 | | 自定义标签 | 作业自定义标签 |
| 节点选择器 | 定义Pod在kubernetes集群上运行的标签选择器,值集中不同表达式值使用逗号分割,例如:`value1,value2`,不同操作符配置方式可参考:https://kubernetes.io/zh-cn/docs/reference/kubernetes-api/common-definitions/node-selector-requirement/ |
| 自定义参数 | kubernetes任务局部的用户自定义参数,自定义参数最终会通过环境变量形式存在于容器中,提供给kubernetes任务使用 | | 自定义参数 | kubernetes任务局部的用户自定义参数,自定义参数最终会通过环境变量形式存在于容器中,提供给kubernetes任务使用 |
## 任务样例 ## 任务样例

3
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java

@ -17,9 +17,11 @@
package org.apache.dolphinscheduler.plugin.task.api.k8s; package org.apache.dolphinscheduler.plugin.task.api.k8s;
import java.util.List;
import java.util.Map; import java.util.Map;
import lombok.Data; import lombok.Data;
import io.fabric8.kubernetes.api.model.NodeSelectorRequirement;
/** /**
* k8s task parameters * k8s task parameters
@ -36,4 +38,5 @@ public class K8sTaskMainParameters {
private double minMemorySpace; private double minMemorySpace;
private Map<String, String> paramsMap; private Map<String, String> paramsMap;
private Map<String, String> labelMap; private Map<String, String> labelMap;
private List<NodeSelectorRequirement> nodeSelectorRequirements;
} }

12
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java

@ -57,6 +57,7 @@ import java.util.concurrent.TimeUnit;
import org.slf4j.Logger; import org.slf4j.Logger;
import io.fabric8.kubernetes.api.model.EnvVar; import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.NodeSelectorTerm;
import io.fabric8.kubernetes.api.model.Quantity; import io.fabric8.kubernetes.api.model.Quantity;
import io.fabric8.kubernetes.api.model.ResourceRequirements; import io.fabric8.kubernetes.api.model.ResourceRequirements;
import io.fabric8.kubernetes.api.model.batch.v1.Job; import io.fabric8.kubernetes.api.model.batch.v1.Job;
@ -125,6 +126,9 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor {
throw new TaskException("Parse yaml-like commands and args failed", e); throw new TaskException("Parse yaml-like commands and args failed", e);
} }
NodeSelectorTerm nodeSelectorTerm = new NodeSelectorTerm();
nodeSelectorTerm.setMatchExpressions(k8STaskMainParameters.getNodeSelectorRequirements());
return new JobBuilder() return new JobBuilder()
.withApiVersion(API_VERSION) .withApiVersion(API_VERSION)
.withNewMetadata() .withNewMetadata()
@ -146,6 +150,14 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor {
.withEnv(envVars) .withEnv(envVars)
.endContainer() .endContainer()
.withRestartPolicy(RESTART_POLICY) .withRestartPolicy(RESTART_POLICY)
.withNewAffinity()
.withNewNodeAffinity()
.withNewRequiredDuringSchedulingIgnoredDuringExecution()
.addNewNodeSelectorTermLike(nodeSelectorTerm)
.endNodeSelectorTerm()
.endRequiredDuringSchedulingIgnoredDuringExecution()
.endNodeAffinity()
.endAffinity()
.endSpec() .endSpec()
.endTemplate() .endTemplate()
.withBackoffLimit(retryNum) .withBackoffLimit(retryNum)

45
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/NodeSelectorExpression.java

@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.model;
import java.io.Serializable;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class NodeSelectorExpression implements Serializable {
/**
* selector key
*/
private String key;
/**
* selector operator
*/
private String operator;
/**
* selector value
*/
private String values;
}

2
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java

@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.plugin.task.api.parameters; package org.apache.dolphinscheduler.plugin.task.api.parameters;
import org.apache.dolphinscheduler.plugin.task.api.model.Label; import org.apache.dolphinscheduler.plugin.task.api.model.Label;
import org.apache.dolphinscheduler.plugin.task.api.model.NodeSelectorExpression;
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
@ -37,6 +38,7 @@ public class K8sTaskParameters extends AbstractParameters {
private String namespace; private String namespace;
private String command; private String command;
private List<Label> customizedLabels; private List<Label> customizedLabels;
private List<NodeSelectorExpression> nodeSelectors;
private String args; private String args;
private double minCpuCores; private double minCpuCores;
private double minMemorySpace; private double minMemorySpace;

8
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java

@ -27,6 +27,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.k8s.impl.K8sTaskExecutor; import org.apache.dolphinscheduler.plugin.task.api.k8s.impl.K8sTaskExecutor;
import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse;
import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
@ -34,6 +35,7 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import io.fabric8.kubernetes.api.model.NodeSelectorRequirement;
import io.fabric8.kubernetes.api.model.batch.v1.Job; import io.fabric8.kubernetes.api.model.batch.v1.Job;
import io.fabric8.kubernetes.api.model.batch.v1.JobStatus; import io.fabric8.kubernetes.api.model.batch.v1.JobStatus;
@ -58,6 +60,11 @@ public class K8sTaskExecutorTest {
String clusterName = namespace.get(CLUSTER); String clusterName = namespace.get(CLUSTER);
Map<String, String> labelMap = new HashMap<>(); Map<String, String> labelMap = new HashMap<>();
labelMap.put("test", "1234"); labelMap.put("test", "1234");
NodeSelectorRequirement requirement = new NodeSelectorRequirement();
requirement.setKey("node-label");
requirement.setOperator("In");
requirement.setValues(Arrays.asList("1234", "123456"));
k8sTaskExecutor = new K8sTaskExecutor(null, taskRequest); k8sTaskExecutor = new K8sTaskExecutor(null, taskRequest);
k8sTaskMainParameters = new K8sTaskMainParameters(); k8sTaskMainParameters = new K8sTaskMainParameters();
k8sTaskMainParameters.setImage(image); k8sTaskMainParameters.setImage(image);
@ -67,6 +74,7 @@ public class K8sTaskExecutorTest {
k8sTaskMainParameters.setMinMemorySpace(minMemorySpace); k8sTaskMainParameters.setMinMemorySpace(minMemorySpace);
k8sTaskMainParameters.setCommand("[\"perl\" ,\"-Mbignum=bpi\", \"-wle\", \"print bpi(2000)\"]"); k8sTaskMainParameters.setCommand("[\"perl\" ,\"-Mbignum=bpi\", \"-wle\", \"print bpi(2000)\"]");
k8sTaskMainParameters.setLabelMap(labelMap); k8sTaskMainParameters.setLabelMap(labelMap);
k8sTaskMainParameters.setNodeSelectorRequirements(Arrays.asList(requirement));
job = k8sTaskExecutor.buildK8sJob(k8sTaskMainParameters); job = k8sTaskExecutor.buildK8sJob(k8sTaskMainParameters);
} }
@Test @Test

28
dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java

@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.plugin.task.k8s; package org.apache.dolphinscheduler.plugin.task.k8s;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.CLUSTER; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.CLUSTER;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.COMMA;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.NAMESPACE_NAME; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.NAMESPACE_NAME;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
@ -26,17 +27,23 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.k8s.AbstractK8sTask; import org.apache.dolphinscheduler.plugin.task.api.k8s.AbstractK8sTask;
import org.apache.dolphinscheduler.plugin.task.api.k8s.K8sTaskMainParameters; import org.apache.dolphinscheduler.plugin.task.api.k8s.K8sTaskMainParameters;
import org.apache.dolphinscheduler.plugin.task.api.model.Label; import org.apache.dolphinscheduler.plugin.task.api.model.Label;
import org.apache.dolphinscheduler.plugin.task.api.model.NodeSelectorExpression;
import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.K8sTaskParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.K8sTaskParameters;
import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.stream.Collectors;
import io.fabric8.kubernetes.api.model.NodeSelectorRequirement;
public class K8sTask extends AbstractK8sTask { public class K8sTask extends AbstractK8sTask {
@ -87,18 +94,33 @@ public class K8sTask extends AbstractK8sTask {
k8sTaskMainParameters.setMinMemorySpace(k8sTaskParameters.getMinMemorySpace()); k8sTaskMainParameters.setMinMemorySpace(k8sTaskParameters.getMinMemorySpace());
k8sTaskMainParameters.setParamsMap(ParamUtils.convert(paramsMap)); k8sTaskMainParameters.setParamsMap(ParamUtils.convert(paramsMap));
k8sTaskMainParameters.setLabelMap(convertToLabelMap(k8sTaskParameters.getCustomizedLabels())); k8sTaskMainParameters.setLabelMap(convertToLabelMap(k8sTaskParameters.getCustomizedLabels()));
k8sTaskMainParameters
.setNodeSelectorRequirements(convertToNodeSelectorRequirements(k8sTaskParameters.getNodeSelectors()));
k8sTaskMainParameters.setCommand(k8sTaskParameters.getCommand()); k8sTaskMainParameters.setCommand(k8sTaskParameters.getCommand());
k8sTaskMainParameters.setArgs(k8sTaskParameters.getArgs()); k8sTaskMainParameters.setArgs(k8sTaskParameters.getArgs());
return JSONUtils.toJsonString(k8sTaskMainParameters); return JSONUtils.toJsonString(k8sTaskMainParameters);
} }
public Map<String, String> convertToLabelMap(List<Label> customizedLabels) { public List<NodeSelectorRequirement> convertToNodeSelectorRequirements(List<NodeSelectorExpression> expressions) {
if (CollectionUtils.isEmpty(customizedLabels)) { if (CollectionUtils.isEmpty(expressions)) {
return Collections.emptyList();
}
return expressions.stream().map(expression -> new NodeSelectorRequirement(
expression.getKey(),
expression.getOperator(),
StringUtils.isEmpty(expression.getValues()) ? Collections.emptyList()
: Arrays.asList(expression.getValues().trim().split(COMMA))))
.collect(Collectors.toList());
}
public Map<String, String> convertToLabelMap(List<Label> labels) {
if (CollectionUtils.isEmpty(labels)) {
return Collections.emptyMap(); return Collections.emptyMap();
} }
HashMap<String, String> labelMap = new HashMap<>(); HashMap<String, String> labelMap = new HashMap<>();
customizedLabels.forEach(label -> { labels.forEach(label -> {
labelMap.put(label.getLabel(), label.getValue()); labelMap.put(label.getLabel(), label.getValue());
}); });
return labelMap; return labelMap;

5
dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sParametersTest.java

@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.plugin.task.k8s; package org.apache.dolphinscheduler.plugin.task.k8s;
import org.apache.dolphinscheduler.plugin.task.api.model.Label; import org.apache.dolphinscheduler.plugin.task.api.model.Label;
import org.apache.dolphinscheduler.plugin.task.api.model.NodeSelectorExpression;
import org.apache.dolphinscheduler.plugin.task.api.parameters.K8sTaskParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.K8sTaskParameters;
import java.util.Arrays; import java.util.Arrays;
@ -37,6 +38,8 @@ public class K8sParametersTest {
private final String command = "[\"/bin/bash\", \"-c\"]"; private final String command = "[\"/bin/bash\", \"-c\"]";
private final String args = "[\"echo hello world\"]"; private final String args = "[\"echo hello world\"]";
private final List<Label> labels = Arrays.asList(new Label("test", "1234")); private final List<Label> labels = Arrays.asList(new Label("test", "1234"));
private final List<NodeSelectorExpression> nodeSelectorExpressions =
Arrays.asList(new NodeSelectorExpression("node-label", "In", "1234,12345"));
@BeforeEach @BeforeEach
public void before() { public void before() {
@ -48,6 +51,7 @@ public class K8sParametersTest {
k8sTaskParameters.setCommand(command); k8sTaskParameters.setCommand(command);
k8sTaskParameters.setArgs(args); k8sTaskParameters.setArgs(args);
k8sTaskParameters.setCustomizedLabels(labels); k8sTaskParameters.setCustomizedLabels(labels);
k8sTaskParameters.setNodeSelectors(nodeSelectorExpressions);
} }
@Test @Test
@ -70,6 +74,7 @@ public class K8sParametersTest {
Assertions.assertEquals(command, k8sTaskParameters.getCommand()); Assertions.assertEquals(command, k8sTaskParameters.getCommand());
Assertions.assertEquals(args, k8sTaskParameters.getArgs()); Assertions.assertEquals(args, k8sTaskParameters.getArgs());
Assertions.assertEquals(labels, k8sTaskParameters.getCustomizedLabels()); Assertions.assertEquals(labels, k8sTaskParameters.getCustomizedLabels());
Assertions.assertEquals(nodeSelectorExpressions, k8sTaskParameters.getNodeSelectors());
} }
} }

8
dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskTest.java

@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
import org.apache.dolphinscheduler.plugin.task.api.model.Label; import org.apache.dolphinscheduler.plugin.task.api.model.Label;
import org.apache.dolphinscheduler.plugin.task.api.model.NodeSelectorExpression;
import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.K8sTaskParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.K8sTaskParameters;
@ -54,6 +55,8 @@ public class K8sTaskTest {
private final String command = "[\"/bin/bash\", \"-c\"]"; private final String command = "[\"/bin/bash\", \"-c\"]";
private final String args = "[\"echo hello world\"]"; private final String args = "[\"echo hello world\"]";
private final List<Label> labels = Arrays.asList(new Label("test", "1234")); private final List<Label> labels = Arrays.asList(new Label("test", "1234"));
private final List<NodeSelectorExpression> nodeSelectorExpressions =
Arrays.asList(new NodeSelectorExpression("node-label", "In", "1234,12345"));
@BeforeEach @BeforeEach
public void before() { public void before() {
@ -65,6 +68,7 @@ public class K8sTaskTest {
k8sTaskParameters.setCommand(command); k8sTaskParameters.setCommand(command);
k8sTaskParameters.setArgs(args); k8sTaskParameters.setArgs(args);
k8sTaskParameters.setCustomizedLabels(labels); k8sTaskParameters.setCustomizedLabels(labels);
k8sTaskParameters.setNodeSelectors(nodeSelectorExpressions);
TaskExecutionContext taskRequest = new TaskExecutionContext(); TaskExecutionContext taskRequest = new TaskExecutionContext();
taskRequest.setTaskInstanceId(taskInstanceId); taskRequest.setTaskInstanceId(taskInstanceId);
taskRequest.setTaskName(taskName); taskRequest.setTaskName(taskName);
@ -90,7 +94,7 @@ public class K8sTaskTest {
@Test @Test
public void testBuildCommandNormal() { public void testBuildCommandNormal() {
String expectedStr = String expectedStr =
"{\"image\":\"ds-dev\",\"command\":\"[\\\"/bin/bash\\\", \\\"-c\\\"]\",\"args\":\"[\\\"echo hello world\\\"]\",\"namespaceName\":\"default\",\"clusterName\":\"lab\",\"minCpuCores\":2.0,\"minMemorySpace\":10.0,\"paramsMap\":{\"day\":\"20220507\"},\"labelMap\":{\"test\":\"1234\"}}"; "{\"image\":\"ds-dev\",\"command\":\"[\\\"/bin/bash\\\", \\\"-c\\\"]\",\"args\":\"[\\\"echo hello world\\\"]\",\"namespaceName\":\"default\",\"clusterName\":\"lab\",\"minCpuCores\":2.0,\"minMemorySpace\":10.0,\"paramsMap\":{\"day\":\"20220507\"},\"labelMap\":{\"test\":\"1234\"},\"nodeSelectorRequirements\":[{\"key\":\"node-label\",\"operator\":\"In\",\"values\":[\"1234\",\"12345\"]}]}";
String commandStr = k8sTask.buildCommand(); String commandStr = k8sTask.buildCommand();
Assertions.assertEquals(expectedStr, commandStr); Assertions.assertEquals(expectedStr, commandStr);
} }
@ -98,7 +102,7 @@ public class K8sTaskTest {
@Test @Test
public void testGetParametersNormal() { public void testGetParametersNormal() {
String expectedStr = String expectedStr =
"K8sTaskParameters(image=ds-dev, namespace={\"name\":\"default\",\"cluster\":\"lab\"}, command=[\"/bin/bash\", \"-c\"], customizedLabels=[Label(label=test, value=1234)], args=[\"echo hello world\"], minCpuCores=2.0, minMemorySpace=10.0)"; "K8sTaskParameters(image=ds-dev, namespace={\"name\":\"default\",\"cluster\":\"lab\"}, command=[\"/bin/bash\", \"-c\"], customizedLabels=[Label(label=test, value=1234)], nodeSelectors=[NodeSelectorExpression(key=node-label, operator=In, values=1234,12345)], args=[\"echo hello world\"], minCpuCores=2.0, minMemorySpace=10.0)";
String result = k8sTask.getParameters().toString(); String result = k8sTask.getParameters().toString();
Assertions.assertEquals(expectedStr, result); Assertions.assertEquals(expectedStr, result);
} }

2
dolphinscheduler-ui/src/components/form/fields/get-field.ts

@ -46,7 +46,7 @@ const getField = (
return widget || null return widget || null
} }
// TODO Support other widgets later // TODO Support other widgets later
if (type === 'custom-parameters' || type === 'custom-labels') { if (type === 'custom-parameters') {
let fieldRules: { [key: string]: IFormItemRule }[] = [] let fieldRules: { [key: string]: IFormItemRule }[] = []
if (rules && !rules[field]) fieldRules = rules[field] = [] if (rules && !rules[field]) fieldRules = rules[field] = []
// @ts-ignore // @ts-ignore

3
dolphinscheduler-ui/src/locales/en_US/project.ts

@ -427,9 +427,12 @@ export default {
value_tips: 'value(optional)', value_tips: 'value(optional)',
value_required_tips: 'value(required)', value_required_tips: 'value(required)',
custom_labels: 'Customized labels', custom_labels: 'Customized labels',
node_selectors: 'Node Selectors',
label_repeat: 'repeated label', label_repeat: 'repeated label',
label_name_tips: 'label name(required)', label_name_tips: 'label name(required)',
label_value_tips: 'label value(required)', label_value_tips: 'label value(required)',
expression_name_tips: 'expression label key(required)',
expression_value_tips: 'expression values(optional)',
pre_tasks: 'Pre tasks', pre_tasks: 'Pre tasks',
program_type: 'Program Type', program_type: 'Program Type',
main_class: 'Main Class', main_class: 'Main Class',

3
dolphinscheduler-ui/src/locales/zh_CN/project.ts

@ -420,9 +420,12 @@ export default {
value_tips: 'value(选填)', value_tips: 'value(选填)',
value_required_tips: 'value(必填)', value_required_tips: 'value(必填)',
custom_labels: '自定义标签', custom_labels: '自定义标签',
node_selectors: '节点选择器',
label_repeat: 'label中有重复', label_repeat: 'label中有重复',
label_name_tips: 'label name(必填)', label_name_tips: 'label name(必填)',
label_value_tips: 'label value(必填)', label_value_tips: 'label value(必填)',
expression_name_tips: '表达式标签键值(必填)',
expression_value_tips: '表达式值集(选填)',
pre_tasks: '前置任务', pre_tasks: '前置任务',
program_type: '程序类型', program_type: '程序类型',
main_class: '主函数的Class', main_class: '主函数的Class',

1
dolphinscheduler-ui/src/views/projects/task/components/node/fields/index.ts

@ -37,6 +37,7 @@ export { useSqlType } from './use-sql-type'
export { useProcedure } from './use-procedure' export { useProcedure } from './use-procedure'
export { useCustomParams } from './use-custom-params' export { useCustomParams } from './use-custom-params'
export { useCustomLabels } from './use-custom-labels' export { useCustomLabels } from './use-custom-labels'
export { useNodeSelectors } from './use-node-selectors'
export { useSourceType } from './use-sqoop-source-type' export { useSourceType } from './use-sqoop-source-type'
export { useTargetType } from './use-sqoop-target-type' export { useTargetType } from './use-sqoop-target-type'
export { useRelationCustomParams } from './use-relation-custom-params' export { useRelationCustomParams } from './use-relation-custom-params'

5
dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-k8s.ts

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
import { useCustomParams, useNamespace, useCustomLabels } from '.' import { useCustomParams, useNamespace, useCustomLabels, useNodeSelectors } from '.'
import type { IJsonItem } from '../types' import type { IJsonItem } from '../types'
import { useI18n } from 'vue-i18n' import { useI18n } from 'vue-i18n'
@ -75,7 +75,8 @@ export function useK8s(model: { [field: string]: any }): IJsonItem[] {
placeholder: t('project.node.args_tips') placeholder: t('project.node.args_tips')
} }
}, },
...useCustomLabels({ model, field: 'customizedLabels' }), ...useCustomLabels({ model, field: 'customizedLabels', name: 'custom_labels' }),
...useNodeSelectors( { model, field: 'nodeSelectors', name: 'node_selectors' }),
...useCustomParams({ model, field: 'localParams', isSimple: true }) ...useCustomParams({ model, field: 'localParams', isSimple: true })
] ]
} }

117
dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-node-selectors.ts

@ -0,0 +1,117 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Ref } from 'vue'
import { useI18n } from 'vue-i18n'
import type { IJsonItem } from '../types'
export function useNodeSelectors({
model,
field,
name,
span = 24
}: {
model: { [field: string]: any }
field: string
name?: string
span?: Ref | number
}): IJsonItem[] {
const { t } = useI18n()
return [
{
type: 'custom-parameters',
field: field,
name: t(`project.node.${name}`),
class: 'btn-custom-parameters',
span,
children: [
{
type: 'input',
field: 'key',
span: 8,
class: 'node-selector-label-name',
props: {
placeholder: t('project.node.expression_name_tips'),
maxLength: 256
},
validate: {
trigger: ['input', 'blur'],
required: true,
validator(validate: any, value: string) {
if (!value) {
return new Error(t('project.node.expression_name_tips'))
}
const sameItems = model[field].filter(
(item: { label: string }) => item.label === value
)
if (sameItems.length > 1) {
return new Error(t('project.node.label_repeat'))
}
}
}
},
{
type: 'select',
field: 'operator',
span: 4,
options: OPERATOR_LIST,
value: 'In',
},
{
type: 'input',
field: 'values',
span: 10,
class: 'node-selector-label-value',
props: {
placeholder: t('project.node.expression_value_tips'),
maxLength: 256,
disabled: false
}
}
]
}
]
}
export const OPERATOR_LIST = [
{
value: 'In',
label: 'In'
},
{
value: 'NotIn',
label: 'NotIn'
},
{
value: 'Exists',
label: 'Exists'
},
{
value: 'DoesNotExist',
label: 'DoesNotExist'
},
{
value: 'Gt',
label: 'Gt'
},
{
value: 'Lt',
label: 'Lt'
}
]

1
dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts

@ -350,6 +350,7 @@ export function formatParams(data: INodeData): {
taskParams.command = data.command taskParams.command = data.command
taskParams.args = data.args taskParams.args = data.args
taskParams.customizedLabels = data.customizedLabels taskParams.customizedLabels = data.customizedLabels
taskParams.nodeSelectors = data.nodeSelectors
} }
if (data.taskType === 'JUPYTER') { if (data.taskType === 'JUPYTER') {

1
dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-k8s.ts

@ -39,6 +39,7 @@ export function useK8s({
timeoutFlag: false, timeoutFlag: false,
localParams: [], localParams: [],
customizedLabels: [], customizedLabels: [],
nodeSelectors: [],
environmentCode: null, environmentCode: null,
failRetryInterval: 1, failRetryInterval: 1,
failRetryTimes: 0, failRetryTimes: 0,

11
dolphinscheduler-ui/src/views/projects/task/components/node/types.ts

@ -66,11 +66,17 @@ interface ILocalParam {
value?: string value?: string
} }
interface ICustomLabel { interface ILabel {
label: string label: string
value: string value: string
} }
interface IMatchExpression {
key: string
operator: string
values: string
}
interface IResponseJsonItem extends Omit<IJsonItemParams, 'type'> { interface IResponseJsonItem extends Omit<IJsonItemParams, 'type'> {
type: 'input' | 'select' | 'radio' | 'group' type: 'input' | 'select' | 'radio' | 'group'
emit: 'change'[] emit: 'change'[]
@ -366,7 +372,8 @@ interface ITaskParams {
image?: string image?: string
command?: string command?: string
args?: string args?: string
customizedLabels?: ICustomLabel[] customizedLabels?: ILabel[]
nodeSelectors?: IMatchExpression[]
algorithm?: string algorithm?: string
params?: string params?: string
searchParams?: string searchParams?: string

Loading…
Cancel
Save