From d324cce6f1432c8c2e5669d6a9c151691bf44dfb Mon Sep 17 00:00:00 2001 From: Tq Date: Mon, 14 Nov 2022 13:52:44 +0800 Subject: [PATCH] [Feature-12526][Api] add dynamic task function: shell task (#12840) * submit dynamic task code * submit shell.json * add dynamic-task-type-config.yaml to standalone server --- .../DynamicTaskTypeConfiguration.java | 94 +++++++++++++++++ .../controller/DynamicTaskTypeController.java | 93 +++++++++++++++++ .../api/dto/taskType/DynamicTaskInfo.java | 38 +++++++ .../src/main/resources/application.yaml | 1 + .../resources/dynamic-task-type-config.yaml | 32 ++++++ .../main/resources/static/shell/shell.json | 98 ++++++++++++++++++ .../src/main/resources/static/shell/shell.png | Bin 0 -> 747 bytes .../resources/static/shell/shell_hover.png | Bin 0 -> 745 bytes .../dolphinscheduler-standalone-server.xml | 7 ++ 9 files changed, 363 insertions(+) create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/DynamicTaskTypeConfiguration.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DynamicTaskTypeController.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/taskType/DynamicTaskInfo.java create mode 100644 dolphinscheduler-api/src/main/resources/dynamic-task-type-config.yaml create mode 100644 dolphinscheduler-api/src/main/resources/static/shell/shell.json create mode 100644 dolphinscheduler-api/src/main/resources/static/shell/shell.png create mode 100644 dolphinscheduler-api/src/main/resources/static/shell/shell_hover.png diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/DynamicTaskTypeConfiguration.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/DynamicTaskTypeConfiguration.java new file mode 100644 index 0000000000..114da5ac36 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/DynamicTaskTypeConfiguration.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.configuration; + +import org.apache.dolphinscheduler.api.dto.taskType.DynamicTaskInfo; +import org.apache.dolphinscheduler.common.config.YamlPropertySourceFactory; +import org.apache.dolphinscheduler.common.constants.Constants; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import lombok.Getter; +import lombok.Setter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.PropertySource; +import org.springframework.stereotype.Component; + +@Component +@EnableConfigurationProperties +@PropertySource(value = {"classpath:dynamic-task-type-config.yaml"}, factory = YamlPropertySourceFactory.class) +@ConfigurationProperties(prefix = "dynamic-task") +@Getter +@Setter +public class DynamicTaskTypeConfiguration { + + private static final Logger logger = LoggerFactory.getLogger(DynamicTaskTypeConfiguration.class); + private static final List defaultTaskCategories = + Arrays.asList(Constants.TYPE_UNIVERSAL, Constants.TYPE_DATA_INTEGRATION, Constants.TYPE_CLOUD, + Constants.TYPE_LOGIC, Constants.TYPE_DATA_QUALITY, Constants.TYPE_OTHER, + Constants.TYPE_MACHINE_LEARNING); + private List universal; + private List cloud; + private List logic; + private List dataIntegration; + private List dataQuality; + private List other; + private List machineLearning; + + public List getTaskCategories() { + return defaultTaskCategories; + } + + public List getTaskTypesByCategory(String category) { + switch (category) { + case Constants.TYPE_UNIVERSAL: + return universal; + case Constants.TYPE_DATA_INTEGRATION: + return cloud; + case Constants.TYPE_CLOUD: + return logic; + case Constants.TYPE_LOGIC: + return dataIntegration; + case Constants.TYPE_DATA_QUALITY: + return dataQuality; + case Constants.TYPE_OTHER: + return other; + case Constants.TYPE_MACHINE_LEARNING: + return machineLearning; + default: + return new ArrayList<>(); + } + + } + + public void printDefaultTypes() { + logger.info("support default universal dynamic task types: {}", universal); + logger.info("support default cloud dynamic task types: {}", cloud); + logger.info("support default logic dynamic task types: {}", logic); + logger.info("support default dataIntegration dynamic task types: {}", dataIntegration); + logger.info("support default dataQuality dynamic task types: {}", dataQuality); + logger.info("support default machineLearning dynamic task types: {}", machineLearning); + logger.info("support default other dynamic task types: {}", other); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DynamicTaskTypeController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DynamicTaskTypeController.java new file mode 100644 index 0000000000..4dd4af6dde --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DynamicTaskTypeController.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.dolphinscheduler.api.controller; + +import static org.apache.dolphinscheduler.api.enums.Status.LIST_TASK_TYPE_ERROR; + +import org.apache.dolphinscheduler.api.aspect.AccessLogAnnotation; +import org.apache.dolphinscheduler.api.configuration.DynamicTaskTypeConfiguration; +import org.apache.dolphinscheduler.api.dto.taskType.DynamicTaskInfo; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.exceptions.ApiException; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.dao.entity.User; + +import java.util.List; + +import javax.annotation.Resource; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; + +/** + * dynamic task type controller + */ +@Tag(name = "DYNAMIC_TASK_TYPE") +@RestController +@RequestMapping("/dynamic") +public class DynamicTaskTypeController extends BaseController { + + @Resource + private DynamicTaskTypeConfiguration dynamicTaskTypeConfiguration; + + /** + * get dynamic task category list + * + * @param loginUser login user + * @return dynamic task category list + */ + @Operation(summary = "listTaskCates", description = "LIST_TASK_TYPE_CATES") + @GetMapping(value = "/taskCategories") + @ResponseStatus(HttpStatus.OK) + @ApiException(LIST_TASK_TYPE_ERROR) + @AccessLogAnnotation(ignoreRequestArgs = "loginUser") + public Result listDynamicTaskCategories(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser) { + List taskCategories = dynamicTaskTypeConfiguration.getTaskCategories(); + return success(Status.SUCCESS.getMsg(), taskCategories); + } + + /** + * get dynamic task category list + * + * @param loginUser login user + * @return dynamic task category list + */ + @Operation(summary = "listDynamicTaskTypes", description = "LIST_DYNAMIC_TASK_TYPES") + @GetMapping(value = "/{taskCategory}/taskTypes") + @ResponseStatus(HttpStatus.OK) + @ApiException(LIST_TASK_TYPE_ERROR) + @AccessLogAnnotation(ignoreRequestArgs = "loginUser") + public Result listDynamicTaskTypes(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @PathVariable("taskCategory") String taskCategory) { + List taskTypes = dynamicTaskTypeConfiguration.getTaskTypesByCategory(taskCategory); + return success(Status.SUCCESS.getMsg(), taskTypes); + } + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/taskType/DynamicTaskInfo.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/taskType/DynamicTaskInfo.java new file mode 100644 index 0000000000..f9319e9730 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/taskType/DynamicTaskInfo.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.dolphinscheduler.api.dto.taskType; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@NoArgsConstructor +@Getter +@Setter +@AllArgsConstructor +public class DynamicTaskInfo { + + private String name; + private String hover; + private String icon; + private String json; + +} diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index f24b158012..1a4ef1cb3f 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -80,6 +80,7 @@ spring: mvc: pathmatch: matching-strategy: ANT_PATH_MATCHER + static-path-pattern: /static/** springdoc: swagger-ui: path: /swagger-ui.html diff --git a/dolphinscheduler-api/src/main/resources/dynamic-task-type-config.yaml b/dolphinscheduler-api/src/main/resources/dynamic-task-type-config.yaml new file mode 100644 index 0000000000..a9a2d55ff8 --- /dev/null +++ b/dolphinscheduler-api/src/main/resources/dynamic-task-type-config.yaml @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +dynamic-task: + universal: + - {name: SHELL,icon: /static/shell/shell-icon.png,hover: /static/shell/shell-hover.png,json: /static/shell/shell.json} + cloud: + - {name: EMR,icon: shell-icon.png,hover: shell-hover.png} + logic: + - {name: SUB_PROCESS,icon: shell-icon.png,hover: shell-hover.png} + dataIntegration: + - {name: SEATUNNEL,icon: shell-icon.png,hover: shell-hover.png} + dataQuality: + - {name: DATA_QUALITY,icon: shell-icon.png,hover: shell-hover.png} + machineLearning: + - {name: JUPYTER,icon: shell-icon.png,hover: shell-hover.png} + other: + - {name: PIGEON,icon: shell-icon.png,hover: shell-hover.png} \ No newline at end of file diff --git a/dolphinscheduler-api/src/main/resources/static/shell/shell.json b/dolphinscheduler-api/src/main/resources/static/shell/shell.json new file mode 100644 index 0000000000..22d3ce7960 --- /dev/null +++ b/dolphinscheduler-api/src/main/resources/static/shell/shell.json @@ -0,0 +1,98 @@ +{ + "task": "shell", + "locales": { + "zh_CN": { + "node_name": "节点名称", + "node_name_tips": "请输入节点名称", + "node_name_validate_message": "节点名称不能为空", + "script_validate_message": "脚本内容不能为空", + "task_priority": "任务优先级", + "highest": "最高", + "high": "高", + "medium": "中", + "low": "低", + "lowest": "最低", + "worker_group": "Worker 分组", + "script": "脚本" + }, + "en_US": { + "node_name": "Node Name", + "node_name_tips": "Please entry node name", + "node_name_validate_message": "Node name cannot be empty", + "script_validate_message": "Script content cannot be empty", + "task_priority": "Task Priority", + "highest": "Highest", + "high": "High", + "medium": "Medium", + "low": "Low", + "lowest": "Lowest", + "worker_group": "Worker Group", + "script": "Script" + } + }, + "apis": { + "getWorkerGroupList": { + "url": "/worker-groups/all", + "method": "get" + } + }, + "forms": [ + { + "label": "task_components.node_name", + "type": "input", + "field": "name", + "defaultValue": "", + "clearable": true, + "placeholder": "task_components.node_name_tips", + "validate": { + "required": true, + "trigger": ["input", "blur"], + "type": "non-empty", + "message": "task_components.node_name_validate_message" + } + }, + { + "label": "task_components.task_priority", + "type": "select", + "field": "taskPriority", + "options": [ + { "label": "task_components.highest", "value": "HIGHEST" }, + { "label": "task_components.high", "value": "HIGH" }, + { "label": "task_components.medium", "value": "MEDIUM" }, + { "label": "task_components.low", "value": "LOW" }, + { "label": "task_components.lowest", "value": "LOWEST" } + ], + "optionsLocale": true, + "defaultValue": "MEDIUM", + "validate": { + "required": true, + "trigger": ["input", "blur"] + } + }, + { + "label": "task_components.worker_group", + "type": "select", + "field": "workerGroup", + "options": [], + "optionsLocale": false, + "defaultValue": "default", + "api": "getWorkerGroupList", + "validate": { + "required": true, + "trigger": ["input", "blur"] + } + }, + { + "label": "task_components.script", + "type": "studio", + "field": "taskParams.rawScript", + "defaultValue": "", + "validate": { + "required": true, + "trigger": ["input", "blur"], + "type": "non-empty", + "message": "task_components.script_validate_message" + } + } + ] +} \ No newline at end of file diff --git a/dolphinscheduler-api/src/main/resources/static/shell/shell.png b/dolphinscheduler-api/src/main/resources/static/shell/shell.png new file mode 100644 index 0000000000000000000000000000000000000000..4e40b6eb2008b68e2c5766e054812a9993380132 GIT binary patch literal 747 zcmeAS@N?(olHy`uVBq!ia0vp^DnP8p!3-p4i=A8uq!^2X+?^QKos)S9Wa|X@gt!6) z|AWE){rhLmoVjh=wnd8;ty;Be?%cUcmMj5^%%4Ah>C&Zh=F9=Ifl{+)&jyMB*$@#R z11K_U)+`_i6akVzjX*)52!snHA+m5~5Gf!D2N3l@F2pn-2?TIih&V(9lZ42kDTWKe z)#G8n^#awvl_8q|7xdc@^#B-N$t6L4!3>N{EUfI@JiL7T0)j#!V&W2V^2!>T+Ioh@ zCgxVwws!UoPA;xKzJ940S$X+I#ieBxmA#YZE?B&L)w=Z?H*Ma!bNAl;2M!-OcKpPd zvllL2y8htNlcz6Vy?y`T)*7s&PRaO{quBj46!(U_e!*JQ=rJPkIS1~ ziaZoAF{*NMF1+CASIHe1!uPS}|N94vwpU!<_U?E5yPs!g*;Zf6NqBKvW17fQr&!@7 zyQVq2b3Xp**k!mN<3wQERUL)f zyzzSuEty!nt>(~C-SirMFaLrQWfBK+wAPCM?OyZtJO@M0*J)2KHGGp`->{2=VZ(8u zx!a!qQhU21O!d0+Zk^xH*|+_3+us-V@8-Sho$oH3)%Fcc%>A){S-x9$+bn*c6dPbj OFnGH9xvXdxEk6 literal 0 HcmV?d00001 diff --git a/dolphinscheduler-api/src/main/resources/static/shell/shell_hover.png b/dolphinscheduler-api/src/main/resources/static/shell/shell_hover.png new file mode 100644 index 0000000000000000000000000000000000000000..b615f5532e35bf3a98783678f64f28af4d2720d7 GIT binary patch literal 745 zcmeAS@N?(olHy`uVBq!ia0vp^Y9P$P3?%12mYf5m7>k44ofy`glX(eb>je0OxB>(&i5T?%*PZrXdk%=LH~qiP1dy!Og#VgT z{_9NouRHm_PXB)`AR8z?>A%Lr|2mUFvN{w0YXcdR{%cP7uQ3Uv2FTTfa3}oNp7dWE zC=S#z5u^ZQ+Qk1FKy!c$2pcF1afmFK1nLDU04W9{Afq3|1qwn`BBa2! zz-19?VCoSfU;`1xf(-|8k*r(!rJxoVUdbgve!&ckOf0PI+&sK|`~rePB4Xkaa`MU= zn%a7X#wO-g*0y%`4o)tvKE8gb8CiMxMa89M6_ve{<}O&geAT-38#isojdhY+t+M%5r-SS>O9DHTnlS8Yg-jF61)W9g_5}D>Ci$g}L3X zzW!Uo4_V#bV{rfR>jl#@*=@d7C-A1*u)Os-D}LwujhRvFcf8zWyfwb~Wh3jRJJ%v2 z|5~0`e7$9tbCL5k?JMs-3vRSO!S>VB^X|vD`_oPt*8j`)nj|(y)avK)uud;2`3Wkf z=Z@AzztNq!EhTD6ieR*w^gHd_U#t_`dqZ!0+&=Bli=UT!7G80g`Zvd(Z(rN2)gPuS Q1H*yA)78&qol`;+08kKhUjP6A literal 0 HcmV?d00001 diff --git a/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml b/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml index 14a38289f8..db312b7298 100644 --- a/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml +++ b/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml @@ -93,6 +93,13 @@ conf + + ${basedir}/../dolphinscheduler-api/src/main/resources + + dynamic-task-type-config.yaml + + conf + ${basedir}/../dolphinscheduler-ui/dist ./ui