[Feature-12526][Api] add dynamic task function: shell task (#12840)

* submit dynamic task code * submit shell.json * add dynamic-task-type-config.yaml to standalone server
2 years ago · d324cce6f1
9 changed files with 363 additions and 0 deletions
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/DynamicTaskTypeConfiguration.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/DynamicTaskTypeConfiguration.java
@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.configuration;
+
+import org.apache.dolphinscheduler.api.dto.taskType.DynamicTaskInfo;
+import org.apache.dolphinscheduler.common.config.YamlPropertySourceFactory;
+import org.apache.dolphinscheduler.common.constants.Constants;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.stereotype.Component;
+
+@Component
+@EnableConfigurationProperties
+@PropertySource(value = {"classpath:dynamic-task-type-config.yaml"}, factory = YamlPropertySourceFactory.class)
+@ConfigurationProperties(prefix = "dynamic-task")
+@Getter
+@Setter
+public class DynamicTaskTypeConfiguration {
+
+    private static final Logger logger = LoggerFactory.getLogger(DynamicTaskTypeConfiguration.class);
+    private static final List<String> defaultTaskCategories =
+            Arrays.asList(Constants.TYPE_UNIVERSAL, Constants.TYPE_DATA_INTEGRATION, Constants.TYPE_CLOUD,
+                    Constants.TYPE_LOGIC, Constants.TYPE_DATA_QUALITY, Constants.TYPE_OTHER,
+                    Constants.TYPE_MACHINE_LEARNING);
+    private List<DynamicTaskInfo> universal;
+    private List<DynamicTaskInfo> cloud;
+    private List<DynamicTaskInfo> logic;
+    private List<DynamicTaskInfo> dataIntegration;
+    private List<DynamicTaskInfo> dataQuality;
+    private List<DynamicTaskInfo> other;
+    private List<DynamicTaskInfo> machineLearning;
+
+    public List<String> getTaskCategories() {
+        return defaultTaskCategories;
+    }
+
+    public List<DynamicTaskInfo> getTaskTypesByCategory(String category) {
+        switch (category) {
+            case Constants.TYPE_UNIVERSAL:
+                return universal;
+            case Constants.TYPE_DATA_INTEGRATION:
+                return cloud;
+            case Constants.TYPE_CLOUD:
+                return logic;
+            case Constants.TYPE_LOGIC:
+                return dataIntegration;
+            case Constants.TYPE_DATA_QUALITY:
+                return dataQuality;
+            case Constants.TYPE_OTHER:
+                return other;
+            case Constants.TYPE_MACHINE_LEARNING:
+                return machineLearning;
+            default:
+                return new ArrayList<>();
+        }
+
+    }
+
+    public void printDefaultTypes() {
+        logger.info("support default universal dynamic task types: {}", universal);
+        logger.info("support default cloud dynamic task types: {}", cloud);
+        logger.info("support default logic dynamic task types: {}", logic);
+        logger.info("support default dataIntegration dynamic task types: {}", dataIntegration);
+        logger.info("support default dataQuality dynamic task types: {}", dataQuality);
+        logger.info("support default machineLearning dynamic task types: {}", machineLearning);
+        logger.info("support default other dynamic task types: {}", other);
+    }
+}
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DynamicTaskTypeController.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DynamicTaskTypeController.java
@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.dolphinscheduler.api.controller;
+
+import static org.apache.dolphinscheduler.api.enums.Status.LIST_TASK_TYPE_ERROR;
+
+import org.apache.dolphinscheduler.api.aspect.AccessLogAnnotation;
+import org.apache.dolphinscheduler.api.configuration.DynamicTaskTypeConfiguration;
+import org.apache.dolphinscheduler.api.dto.taskType.DynamicTaskInfo;
+import org.apache.dolphinscheduler.api.enums.Status;
+import org.apache.dolphinscheduler.api.exceptions.ApiException;
+import org.apache.dolphinscheduler.api.utils.Result;
+import org.apache.dolphinscheduler.common.constants.Constants;
+import org.apache.dolphinscheduler.dao.entity.User;
+
+import java.util.List;
+
+import javax.annotation.Resource;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestAttribute;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.ResponseStatus;
+import org.springframework.web.bind.annotation.RestController;
+
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.tags.Tag;
+
+/**
+ * dynamic task type controller
+ */
+@Tag(name = "DYNAMIC_TASK_TYPE")
+@RestController
+@RequestMapping("/dynamic")
+public class DynamicTaskTypeController extends BaseController {
+
+    @Resource
+    private DynamicTaskTypeConfiguration dynamicTaskTypeConfiguration;
+
+    /**
+     * get dynamic task category list
+     *
+     * @param loginUser login user
+     * @return dynamic task category list
+     */
+    @Operation(summary = "listTaskCates", description = "LIST_TASK_TYPE_CATES")
+    @GetMapping(value = "/taskCategories")
+    @ResponseStatus(HttpStatus.OK)
+    @ApiException(LIST_TASK_TYPE_ERROR)
+    @AccessLogAnnotation(ignoreRequestArgs = "loginUser")
+    public Result listDynamicTaskCategories(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser) {
+        List<String> taskCategories = dynamicTaskTypeConfiguration.getTaskCategories();
+        return success(Status.SUCCESS.getMsg(), taskCategories);
+    }
+
+    /**
+     * get dynamic task category list
+     *
+     * @param loginUser login user
+     * @return dynamic task category list
+     */
+    @Operation(summary = "listDynamicTaskTypes", description = "LIST_DYNAMIC_TASK_TYPES")
+    @GetMapping(value = "/{taskCategory}/taskTypes")
+    @ResponseStatus(HttpStatus.OK)
+    @ApiException(LIST_TASK_TYPE_ERROR)
+    @AccessLogAnnotation(ignoreRequestArgs = "loginUser")
+    public Result listDynamicTaskTypes(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
+                                       @PathVariable("taskCategory") String taskCategory) {
+        List<DynamicTaskInfo> taskTypes = dynamicTaskTypeConfiguration.getTaskTypesByCategory(taskCategory);
+        return success(Status.SUCCESS.getMsg(), taskTypes);
+    }
+
+}
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/taskType/DynamicTaskInfo.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/taskType/DynamicTaskInfo.java
@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.dolphinscheduler.api.dto.taskType;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@NoArgsConstructor
+@Getter
+@Setter
+@AllArgsConstructor
+public class DynamicTaskInfo {
+
+    private String name;
+    private String hover;
+    private String icon;
+    private String json;
+
+}
--- a/dolphinscheduler-api/src/main/resources/application.yaml
+++ b/dolphinscheduler-api/src/main/resources/application.yaml
@ -80,6 +80,7 @@ spring:
  mvc:
    pathmatch:
      matching-strategy: ANT_PATH_MATCHER
+    static-path-pattern: /static/**
 springdoc:
  swagger-ui:
    path: /swagger-ui.html
--- a/dolphinscheduler-api/src/main/resources/dynamic-task-type-config.yaml
+++ b/dolphinscheduler-api/src/main/resources/dynamic-task-type-config.yaml
@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dynamic-task:
+  universal:
+    - {name: SHELL,icon: /static/shell/shell-icon.png,hover: /static/shell/shell-hover.png,json: /static/shell/shell.json}
+  cloud:
+    - {name: EMR,icon: shell-icon.png,hover: shell-hover.png}
+  logic:
+    - {name: SUB_PROCESS,icon: shell-icon.png,hover: shell-hover.png}
+  dataIntegration:
+    - {name: SEATUNNEL,icon: shell-icon.png,hover: shell-hover.png}
+  dataQuality:
+    - {name: DATA_QUALITY,icon: shell-icon.png,hover: shell-hover.png}
+  machineLearning:
+    - {name: JUPYTER,icon: shell-icon.png,hover: shell-hover.png}
+  other:
+    - {name: PIGEON,icon: shell-icon.png,hover: shell-hover.png}
--- a/dolphinscheduler-api/src/main/resources/static/shell/shell.json
+++ b/dolphinscheduler-api/src/main/resources/static/shell/shell.json
@ -0,0 +1,98 @@
+{
+  "task": "shell",
+  "locales": {
+    "zh_CN": {
+      "node_name": "节点名称",
+      "node_name_tips": "请输入节点名称",
+      "node_name_validate_message": "节点名称不能为空",
+      "script_validate_message": "脚本内容不能为空",
+      "task_priority": "任务优先级",
+      "highest": "最高",
+      "high": "高",
+      "medium": "中",
+      "low": "低",
+      "lowest": "最低",
+      "worker_group": "Worker 分组",
+      "script": "脚本"
+    },
+    "en_US": {
+      "node_name": "Node Name",
+      "node_name_tips": "Please entry node name",
+      "node_name_validate_message": "Node name cannot be empty",
+      "script_validate_message": "Script content cannot be empty",
+      "task_priority": "Task Priority",
+      "highest": "Highest",
+      "high": "High",
+      "medium": "Medium",
+      "low": "Low",
+      "lowest": "Lowest",
+      "worker_group": "Worker Group",
+      "script": "Script"
+    }
+  },
+  "apis": {
+    "getWorkerGroupList": {
+      "url": "/worker-groups/all",
+      "method": "get"
+    }
+  },
+  "forms": [
+    {
+      "label": "task_components.node_name",
+      "type": "input",
+      "field": "name",
+      "defaultValue": "",
+      "clearable": true,
+      "placeholder": "task_components.node_name_tips",
+      "validate": {
+        "required": true,
+        "trigger": ["input", "blur"],
+        "type": "non-empty",
+        "message": "task_components.node_name_validate_message"
+      }
+    },
+    {
+      "label": "task_components.task_priority",
+      "type": "select",
+      "field": "taskPriority",
+      "options": [
+        { "label": "task_components.highest", "value": "HIGHEST" },
+        { "label": "task_components.high", "value": "HIGH" },
+        { "label": "task_components.medium", "value": "MEDIUM" },
+        { "label": "task_components.low", "value": "LOW" },
+        { "label": "task_components.lowest", "value": "LOWEST" }
+      ],
+      "optionsLocale": true,
+      "defaultValue": "MEDIUM",
+      "validate": {
+        "required": true,
+        "trigger": ["input", "blur"]
+      }
+    },
+    {
+      "label": "task_components.worker_group",
+      "type": "select",
+      "field": "workerGroup",
+      "options": [],
+      "optionsLocale": false,
+      "defaultValue": "default",
+      "api": "getWorkerGroupList",
+      "validate": {
+        "required": true,
+        "trigger": ["input", "blur"]
+      }
+    },
+    {
+      "label": "task_components.script",
+      "type": "studio",
+      "field": "taskParams.rawScript",
+      "defaultValue": "",
+      "validate": {
+        "required": true,
+        "trigger": ["input", "blur"],
+        "type": "non-empty",
+        "message": "task_components.script_validate_message"
+      }
+    }
+  ]
+}
--- a/dolphinscheduler-api/src/main/resources/static/shell/shell.png
+++ b/dolphinscheduler-api/src/main/resources/static/shell/shell.png
--- a/dolphinscheduler-api/src/main/resources/static/shell/shell_hover.png
+++ b/dolphinscheduler-api/src/main/resources/static/shell/shell_hover.png
--- a/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml
+++ b/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml
@ -93,6 +93,13 @@
            </includes>
            <outputDirectory>conf</outputDirectory>
        </fileSet>
+        <fileSet>
+            <directory>${basedir}/../dolphinscheduler-api/src/main/resources</directory>
+            <includes>
+                <include>dynamic-task-type-config.yaml</include>
+            </includes>
+            <outputDirectory>conf</outputDirectory>
+        </fileSet>
        <fileSet>
            <directory>${basedir}/../dolphinscheduler-ui/dist</directory>
            <outputDirectory>./ui</outputDirectory>