diff --git a/docs/configs/docsdev.js b/docs/configs/docsdev.js index b69a7f3354..d7cac77fa9 100644 --- a/docs/configs/docsdev.js +++ b/docs/configs/docsdev.js @@ -161,6 +161,10 @@ export default { title: 'Jupyter', link: '/en-us/docs/dev/user_doc/guide/task/jupyter.html', }, + { + title: 'Hive CLI', + link: '/en-us/docs/dev/user_doc/guide/task/hive-cli.html', + }, { title: 'Kubernetes', link: '/en-us/docs/dev/user_doc/guide/task/kubernetes.html', @@ -781,6 +785,10 @@ export default { title: 'Jupyter', link: '/zh-cn/docs/dev/user_doc/guide/task/jupyter.html', }, + { + title: 'Hive CLI', + link: '/zh-cn/docs/dev/user_doc/guide/task/hive-cli.html', + }, { title: 'Kubernetes', link: '/zh-cn/docs/dev/user_doc/guide/task/kubernetes.html', diff --git a/docs/docs/en/guide/task/hive-cli.md b/docs/docs/en/guide/task/hive-cli.md new file mode 100644 index 0000000000..556040150d --- /dev/null +++ b/docs/docs/en/guide/task/hive-cli.md @@ -0,0 +1,56 @@ +# Hive CLI + +## Overview + +Use `Hive Cli Task` to create a `Hive Cli` type task and execute hive SQL from scripts or files. +The workers run `hive -e` to execute hive sql from scripts or `hive -f` to execute from files in `Resource Center`. + +## Hive CLI Task vs SQL Task With Hive Datasource + +In DolphinScheduler, we have both `Hive CLI Task` and `SQL Task With Hive Datasource` for different scenarios. +You could choose between these two based on your needs. + +- The `Hive CLI` task plugin connects directly to `HDFS` and the `Hive Metastore` for hive task executions, + which requires your workers to have access to those services, such as related `Hive` libs, `Hive` and `HDFS` configuration files. + However, `Hive CLI Task` provides better stability for scheduling in production. +- `SQL Task With Hive Datasource` does not require access to `Hive` libs, `Hive` and + `HDFS` configuration files and supports `Kerberos` for authentication. However, you may encounter `HiveServer2` failures + if your hive sql task scheduling puts significant pressure on it. + +## Create Task + +- Click `Project Management-Project Name-Workflow Definition`, and click the `Create Workflow` button to enter the DAG editing page. +- Drag from the toolbar to the canvas. + +## Task Parameters + +| **Parameter** | **Description** | +|------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Node Name | The name of the task. Node names within the same workflow must be unique. | +| Run Flag | Indicating whether to schedule the task. If you do not need to execute the task, you can turn on the `Prohibition execution` switch. | +| Description | Describing the function of this node. | +| Task Priority | When the number of the worker threads is insufficient, the worker executes task according to the priority. When two tasks have the same priority, the worker will execute them in `first come first served` fashion. | +| Worker Group | Machines which execute the tasks. If you choose `default`, scheduler will send the task to a random worker. | +| Task Group Name | Resource group of tasks. It will not take effect if not configured. | +| Environment Name | Environment to execute the task. | +| Number of Failed Retries | The number of task retries for failures. You could select it by drop-down menu or fill it manually. | +| Failure Retry Interval | Interval of task retries for failures. You could select it by drop-down menu or fill it manually. | +| CPU Quota | Assign the specified CPU time quota to the task executed. Takes a percentage value. Default -1 means unlimited. For example, the full CPU load of one core is 100%, and that of 16 cores is 1600%. You could configure it by [task.resource.limit.state](../../architecture/configuration.md). | +| Max Memory | Assign the specified max memory to the task executed. Exceeding this limit will trigger oom to be killed and will not automatically retry. Takes an MB value. Default -1 means unlimited. You could configure it by [task.resource.limit.state](../../architecture/configuration.md). | +| Timeout Alarm | Alarm for task timeout. When the task exceeds the "timeout threshold", an alarm email will send. | +| Hive Cli Task Execution Type | The type of hive cli task execution, choose either `FROM_SCRIPT` or `FROM_FILE`. | +| Hive SQL Script | If you choose `FROM_SCRIPT` for `Hive Cli Task Execution Type`, you need to fill in your SQL script. | +| Hive Cli Options | Extra options for hive cli, such as `--verbose` | +| Resources | If you choose `FROM_FILE` for `Hive Cli Task Execution Type`, you need to select your SQL file. | + +## Task Example + +### Hive Cli Task Example + +This example below illustrates how to create a `Hive CLI` task node and execute hive SQL from script: + +![demo-hive-cli-from-script](../../../../img/tasks/demo/hive_cli_from_script.png) + +This example below illustrates how to create a `Hive CLI` task node and execute hive SQL from file: + +![demo-hive-cli-from-file](../../../../img/tasks/demo/hive_cli_from_file.png) diff --git a/docs/docs/zh/guide/task/hive-cli.md b/docs/docs/zh/guide/task/hive-cli.md new file mode 100644 index 0000000000..54b5e4db12 --- /dev/null +++ b/docs/docs/zh/guide/task/hive-cli.md @@ -0,0 +1,57 @@ +# Hive CLI + +## 综述 + +使用`Hive Cli任务插件`创建`Hive Cli`类型的任务执行SQL脚本语句或者SQL任务文件。 +执行任务的worker会通过`hive -e`命令执行hive SQL脚本语句或者通过`hive -f`命令执行`资源中心`中的hive SQL文件。 + +## Hive CLI任务 VS 连接Hive数据源的SQL任务 + +在DolphinScheduler中,我们有`Hive CLI任务插件`和`使用Hive数据源的SQL插件`提供用户在不同场景下使用,您可以根据需要进行选择。 + +- `Hive CLI任务插件`直接连接`HDFS`和`Hive Metastore`来执行hive类型的任务,所以需要能够访问到对应的服务。 + 执行任务的worker节点需要有相应的`Hive` jar包以及`Hive`和`HDFS`的配置文件。 + 但是在生产调度中,`Hive CLI任务插件`能够提供更可靠的稳定性。 +- `使用Hive数据源的SQL插件`不需要您在worker节点上有相应的`Hive` jar包以及`Hive`和`HDFS`的配置文件,而且支持 `Kerberos`认证。 + 但是在生产调度中,若调度压力很大,使用这种方式可能会遇到`HiveServer2`服务过载失败等问题。 + +## 创建任务 + +- 点击项目管理-项目名称-工作流定义,点击"创建工作流"按钮,进入DAG编辑页面。 +- 工具栏中拖动 到画板中,即可完成创建。 + +## 任务参数 + +- 前置任务:选择当前任务的前置任务,会将被选择的前置任务设置为当前任务的上游。 + +| **任务参数** | **描述** | +|---------------|-------------------------------------------------------------------------------------------------------------------------------------| +| 任务名称 | 设置任务的名称。一个工作流定义中的节点名称是唯一的。 | +| 运行标志 | 标识这个节点是否需要正常调度,如果不需要执行,可以打开禁止执行开关。 | +| 描述 | 描述该节点的功能。 | +| 任务优先级 | worker线程数不足时,根据优先级从高到低依次执行,优先级一样时根据先进先出原则执行。 | +| Worker分组 | 任务分配给worker组的机器机执行,选择Default,会随机选择一台worker机执行。 | +| 任务组名称 | 任务资源组,如果没有配置的话就不会生效。 | +| 环境名称 | 配置任务执行的环境。 | +| 失败重试次数 | 任务失败重新提交的次数,支持下拉和手填。 | +| 失败重试间隔 | 任务失败重新提交任务的时间间隔,支持下拉和手填。 | +| CPU 配额 | 为执行的任务分配指定的CPU时间配额,单位百分比,默认-1代表不限制,例如1个核心的CPU满载是100%,16个核心的是1600%。 [task.resource.limit.state](../../architecture/configuration.md) | +| 最大内存 | 为执行的任务分配指定的内存大小,超过会触发OOM被Kill同时不会进行自动重试,单位MB,默认-1代表不限制。这个功能由 [task.resource.limit.state](../../architecture/configuration.md) 控制。 | +| 超时告警 | 勾选超时告警、超时失败,当任务超过"超时时长"后,会发送告警邮件并且任务执行失败.这个功能由 [task.resource.limit.state](../../architecture/configuration.md) 控制。 | +| Hive Cli 任务类型 | Hive Cli任务执行方式,可以选择`FROM_SCRIPT`或者`FROM_FILE`。 | +| Hive SQL 脚本 | 手动填入您的Hive SQL脚本语句。 | +| Hive Cli 选项 | Hive Cli的其他选项,如`--verbose`。 | +| 资源 | 如果您选择`FROM_FILE`作为Hive Cli任务类型,您需要在资源中选择Hive SQL文件。 | + +## 任务样例 + +### Hive CLI任务样例 + +下面的样例演示了如何使用`Hive CLI`任务节点执行Hive SQL脚本语句: + +![demo-hive-cli-from-script](../../../../img/tasks/demo/hive_cli_from_script.png) + +下面的样例演示了如何使用`Hive CLI`任务节点从资源中心的Hive SQL + +![demo-hive-cli-from-file](../../../../img/tasks/demo/hive_cli_from_file.png) + diff --git a/docs/img/tasks/demo/hive_cli_from_file.png b/docs/img/tasks/demo/hive_cli_from_file.png new file mode 100644 index 0000000000..164a8095f2 Binary files /dev/null and b/docs/img/tasks/demo/hive_cli_from_file.png differ diff --git a/docs/img/tasks/demo/hive_cli_from_script.png b/docs/img/tasks/demo/hive_cli_from_script.png new file mode 100644 index 0000000000..46fe288545 Binary files /dev/null and b/docs/img/tasks/demo/hive_cli_from_script.png differ diff --git a/docs/img/tasks/icons/hivecli.png b/docs/img/tasks/icons/hivecli.png new file mode 100644 index 0000000000..a363bb3d36 Binary files /dev/null and b/docs/img/tasks/icons/hivecli.png differ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml index fd1111d658..665d28783d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-all @@ -201,6 +200,12 @@ dolphinscheduler-task-pytorch ${project.version} + + + org.apache.dolphinscheduler + dolphinscheduler-task-hivecli + ${project.version} + - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/pom.xml new file mode 100644 index 0000000000..ab458f0e97 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/pom.xml @@ -0,0 +1,40 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-task-plugin + dev-SNAPSHOT + + + dolphinscheduler-task-hivecli + jar + + + + org.apache.dolphinscheduler + dolphinscheduler-spi + + + org.apache.dolphinscheduler + dolphinscheduler-task-api + + + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliConstants.java new file mode 100644 index 0000000000..8b92999241 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliConstants.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.hivecli; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public class HiveCliConstants { + + public static final String TYPE_SCRIPT = "SCRIPT"; + + public static final String TYPE_FILE = "FILE"; + + public static final String HIVE_CLI_EXECUTE_FILE = "hive -f"; + + public static final String HIVE_CLI_EXECUTE_SCRIPT = "hive -e \"%s\""; + +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliParameters.java new file mode 100644 index 0000000000..f2be1dd897 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliParameters.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.hivecli; + +import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import java.util.List; + +import lombok.Data; + +@Data +public class HiveCliParameters extends AbstractParameters { + + private String hiveSqlScript; + + private String hiveCliTaskExecutionType; + + private String hiveCliOptions; + + private List resourceList; + + @Override + public boolean checkParameters() { + if (!StringUtils.isNotEmpty(hiveCliTaskExecutionType)) { + return false; + } + + if (HiveCliConstants.TYPE_SCRIPT.equals(hiveCliTaskExecutionType)) { + return StringUtils.isNotEmpty(hiveSqlScript); + } else if (HiveCliConstants.TYPE_FILE.equals(hiveCliTaskExecutionType)) { + return (resourceList != null) && (resourceList.size() > 0); + } else { + return false; + } + + } + + @Override + public List getResourceFilesList() { + return this.resourceList; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTask.java new file mode 100644 index 0000000000..2af1bacb38 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTask.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.hivecli; + +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; + +import org.apache.dolphinscheduler.plugin.task.api.AbstractTaskExecutor; +import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor; +import org.apache.dolphinscheduler.plugin.task.api.TaskException; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; +import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; +import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; +import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; + +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class HiveCliTask extends AbstractTaskExecutor { + + private HiveCliParameters hiveCliParameters; + + private final ShellCommandExecutor shellCommandExecutor; + + private final TaskExecutionContext taskExecutionContext; + + public HiveCliTask(TaskExecutionContext taskExecutionContext) { + super(taskExecutionContext); + this.taskExecutionContext = taskExecutionContext; + + this.shellCommandExecutor = new ShellCommandExecutor(this::logHandle, + taskExecutionContext, + logger); + } + + @Override + public void init() { + logger.info("hiveCli task params {}", taskExecutionContext.getTaskParams()); + + hiveCliParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), HiveCliParameters.class); + + if (!hiveCliParameters.checkParameters()) { + throw new TaskException("hiveCli task params is not valid"); + } + } + + @Override + public void handle() throws TaskException { + try { + final TaskResponse taskResponse = shellCommandExecutor.run(buildCommand()); + setExitStatusCode(taskResponse.getExitStatusCode()); + setAppIds(taskResponse.getAppIds()); + setProcessId(taskResponse.getProcessId()); + setVarPool(shellCommandExecutor.getVarPool()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.error("The current HiveCLI Task has been interrupted", e); + setExitStatusCode(EXIT_CODE_FAILURE); + throw new TaskException("The current HiveCLI Task has been interrupted", e); + } catch (Exception e) { + logger.error("hiveCli task failure", e); + setExitStatusCode(EXIT_CODE_FAILURE); + throw new TaskException("run hiveCli task error", e); + } + } + + protected String buildCommand() { + + final List args = new ArrayList<>(); + + final String type = hiveCliParameters.getHiveCliTaskExecutionType(); + + // TODO: make sure type is not unknown + if (HiveCliConstants.TYPE_FILE.equals(type)) { + args.add(HiveCliConstants.HIVE_CLI_EXECUTE_FILE); + final List resourceInfos = hiveCliParameters.getResourceList(); + if (resourceInfos.size() > 1) { + logger.warn("more than 1 files detected, use the first one by default"); + } + + args.add(StringUtils.stripStart(resourceInfos.get(0).getResourceName(), "/")); + } else { + final String script = hiveCliParameters.getHiveSqlScript(); + args.add(String.format(HiveCliConstants.HIVE_CLI_EXECUTE_SCRIPT, script)); + } + + final String hiveCliOptions = hiveCliParameters.getHiveCliOptions(); + if (StringUtils.isNotEmpty(hiveCliOptions)) { + args.add(hiveCliOptions); + } + + final Map paramsMap = taskExecutionContext.getPrepareParamsMap(); + final String command = + ParameterUtils.convertParameterPlaceholders(String.join(" ", args), ParamUtils.convert(paramsMap)); + + logger.info("hiveCli task command: {}", command); + + return command; + + } + + @Override + public AbstractParameters getParameters() { + return hiveCliParameters; + } + + @Override + public void cancelApplication(boolean cancelApplication) throws Exception { + shellCommandExecutor.cancelApplication(); + } + +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskChannel.java new file mode 100644 index 0000000000..f9b6726dc8 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskChannel.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.hivecli; + +import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; +import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; +import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; + +public class HiveCliTaskChannel implements TaskChannel { + + @Override + public void cancelApplication(boolean status) { + + } + + @Override + public AbstractTask createTask(TaskExecutionContext taskExecutionContext) { + return new HiveCliTask(taskExecutionContext); + } + + @Override + public AbstractParameters parseParameters(ParametersNode parametersNode) { + return JSONUtils.parseObject(parametersNode.getTaskParams(), HiveCliParameters.class); + } + + @Override + public ResourceParametersHelper getResources(String parameters) { + return null; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskChannelFactory.java new file mode 100644 index 0000000000..244ac2eba3 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskChannelFactory.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.hivecli; + +import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; +import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; +import org.apache.dolphinscheduler.spi.params.base.PluginParams; + +import java.util.List; + +import com.google.auto.service.AutoService; + +@AutoService(TaskChannelFactory.class) +public class HiveCliTaskChannelFactory implements TaskChannelFactory { + + @Override + public TaskChannel create() { + return new HiveCliTaskChannel(); + } + + @Override + public String getName() { + return "HIVECLI"; + } + + @Override + public List getParams() { + return null; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java new file mode 100644 index 0000000000..62655dfb86 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.hivecli; + +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class HiveCliTaskTest { + + public static final String EXPECTED_HIVE_CLI_TASK_EXECUTE_FROM_SCRIPT_COMMAND = + "hive -e \"SHOW DATABASES;\""; + + public static final String EXPECTED_HIVE_CLI_TASK_EXECUTE_FROM_FILE_COMMAND = + "hive -f sql_tasks/hive_task.sql"; + + public static final String EXPECTED_HIVE_CLI_TASK_EXECUTE_WITH_OPTIONS = + "hive -e \"SHOW DATABASES;\" --verbose"; + + @Test + public void hiveCliTaskExecuteSqlFromScript() throws Exception { + String hiveCliTaskParameters = buildHiveCliTaskExecuteSqlFromScriptParameters(); + HiveCliTask hiveCliTask = prepareHiveCliTaskForTest(hiveCliTaskParameters); + hiveCliTask.init(); + Assert.assertEquals(hiveCliTask.buildCommand(), EXPECTED_HIVE_CLI_TASK_EXECUTE_FROM_SCRIPT_COMMAND); + } + + @Test + public void hiveCliTaskExecuteSqlFromFile() throws Exception { + String hiveCliTaskParameters = buildHiveCliTaskExecuteSqlFromFileParameters(); + HiveCliTask hiveCliTask = prepareHiveCliTaskForTest(hiveCliTaskParameters); + hiveCliTask.init(); + Assert.assertEquals(hiveCliTask.buildCommand(), EXPECTED_HIVE_CLI_TASK_EXECUTE_FROM_FILE_COMMAND); + } + + @Test + public void hiveCliTaskExecuteWithOptions() throws Exception { + String hiveCliTaskParameters = buildHiveCliTaskExecuteWithOptionsParameters(); + HiveCliTask hiveCliTask = prepareHiveCliTaskForTest(hiveCliTaskParameters); + hiveCliTask.init(); + Assert.assertEquals(hiveCliTask.buildCommand(), EXPECTED_HIVE_CLI_TASK_EXECUTE_WITH_OPTIONS); + } + + private HiveCliTask prepareHiveCliTaskForTest(final String hiveCliTaskParameters) { + TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class); + when(taskExecutionContext.getTaskParams()).thenReturn(hiveCliTaskParameters); + HiveCliTask hiveCliTask = spy(new HiveCliTask(taskExecutionContext)); + return hiveCliTask; + } + + private String buildHiveCliTaskExecuteSqlFromScriptParameters() { + final HiveCliParameters hiveCliParameters = new HiveCliParameters(); + hiveCliParameters.setHiveCliTaskExecutionType("SCRIPT"); + hiveCliParameters.setHiveSqlScript("SHOW DATABASES;"); + return JSONUtils.toJsonString(hiveCliParameters); + } + + private String buildHiveCliTaskExecuteSqlFromFileParameters() { + final HiveCliParameters hiveCliParameters = new HiveCliParameters(); + hiveCliParameters.setHiveCliTaskExecutionType("FILE"); + List resources = new ArrayList<>(); + ResourceInfo sqlResource = new ResourceInfo(); + sqlResource.setResourceName("/sql_tasks/hive_task.sql"); + resources.add(sqlResource); + hiveCliParameters.setResourceList(resources); + return JSONUtils.toJsonString(hiveCliParameters); + } + + private String buildHiveCliTaskExecuteWithOptionsParameters() { + final HiveCliParameters hiveCliParameters = new HiveCliParameters(); + hiveCliParameters.setHiveCliTaskExecutionType("SCRIPT"); + hiveCliParameters.setHiveSqlScript("SHOW DATABASES;"); + hiveCliParameters.setHiveCliOptions("--verbose"); + return JSONUtils.toJsonString(hiveCliParameters); + } + +} diff --git a/dolphinscheduler-task-plugin/pom.xml b/dolphinscheduler-task-plugin/pom.xml index 35444c62ad..528e21f459 100644 --- a/dolphinscheduler-task-plugin/pom.xml +++ b/dolphinscheduler-task-plugin/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler org.apache.dolphinscheduler + dolphinscheduler dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-plugin pom @@ -61,6 +60,7 @@ dolphinscheduler-task-chunjun dolphinscheduler-task-flink-stream dolphinscheduler-task-pytorch + dolphinscheduler-task-hivecli @@ -74,4 +74,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-ui/public/images/task-icons/hivecli.png b/dolphinscheduler-ui/public/images/task-icons/hivecli.png new file mode 100644 index 0000000000..9113cbd8ae Binary files /dev/null and b/dolphinscheduler-ui/public/images/task-icons/hivecli.png differ diff --git a/dolphinscheduler-ui/public/images/task-icons/hivecli_hover.png b/dolphinscheduler-ui/public/images/task-icons/hivecli_hover.png new file mode 100644 index 0000000000..7e533aa6e9 Binary files /dev/null and b/dolphinscheduler-ui/public/images/task-icons/hivecli_hover.png differ diff --git a/dolphinscheduler-ui/src/layouts/content/use-dataList.ts b/dolphinscheduler-ui/src/layouts/content/use-dataList.ts index 37794158b8..0d8fb17731 100644 --- a/dolphinscheduler-ui/src/layouts/content/use-dataList.ts +++ b/dolphinscheduler-ui/src/layouts/content/use-dataList.ts @@ -341,7 +341,9 @@ export function useDataList() { label: t('user_dropdown.password'), key: 'password', icon: renderIcon(KeyOutlined), - disabled: (userStore.getUserInfo as UserInfoRes).securityConfigType !== 'PASSWORD' + disabled: + (userStore.getUserInfo as UserInfoRes).securityConfigType !== + 'PASSWORD' }, { label: t('user_dropdown.logout'), diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts index f8751df123..fa599a466c 100644 --- a/dolphinscheduler-ui/src/locales/en_US/project.ts +++ b/dolphinscheduler-ui/src/locales/en_US/project.ts @@ -646,11 +646,20 @@ export default { zeppelin_paragraph_id_tips: 'Please enter the paragraph id of your zeppelin paragraph', zeppelin_parameters: 'parameters', - zeppelin_parameters_tips: 'Please enter the parameters for zeppelin dynamic form', + zeppelin_parameters_tips: + 'Please enter the parameters for zeppelin dynamic form', zeppelin_rest_endpoint: 'zeppelinRestEndpoint', - zeppelin_rest_endpoint_tips: 'Please enter the rest endpoint of your Zeppelin server', - zeppelin_production_note_directory: 'Directory for cloned zeppelin note in production mode', - zeppelin_production_note_directory_tips: 'Please enter the production note directory to enable production mode', + zeppelin_rest_endpoint_tips: + 'Please enter the rest endpoint of your Zeppelin server', + zeppelin_production_note_directory: + 'Directory for cloned zeppelin note in production mode', + zeppelin_production_note_directory_tips: + 'Please enter the production note directory to enable production mode', + hive_cli_task_execution_type: 'Hive Cli Task Execution Type', + hive_sql_script: 'Hive SQL Script', + hive_cli_options: 'Hive Cli Options', + hive_cli_options_tips: + 'Please enter the options for hive cli, e.g. --verbose', jupyter_conda_env_name: 'condaEnvName', jupyter_conda_env_name_tips: 'Please enter the conda environment name of papermill', @@ -764,6 +773,7 @@ export default { pytorch_python_env_tool: 'Python Environment Manager Tool', pytorch_requirements: 'Requirement File', pytorch_conda_python_version: 'Python Version', - pytorch_conda_python_version_tips: 'Please enter the version number, such as 3.6, 3.7, 3.x' + pytorch_conda_python_version_tips: + 'Please enter the version number, such as 3.6, 3.7, 3.x' } } diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts index 6f423786ba..a08d0dc678 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts @@ -638,12 +638,17 @@ export default { zeppelin_note_id_tips: '请输入zeppelin note id', zeppelin_paragraph_id: 'zeppelinParagraphId', zeppelin_production_note_directory: '生产模式下存放克隆note的目录', - zeppelin_production_note_directory_tips: '请输入生产环境note目录以启用生产模式', + zeppelin_production_note_directory_tips: + '请输入生产环境note目录以启用生产模式', zeppelin_paragraph_id_tips: '请输入zeppelin paragraph id', zeppelin_parameters: 'parameters', zeppelin_parameters_tips: '请输入zeppelin dynamic form参数', zeppelin_rest_endpoint: 'zeppelinRestEndpoint', zeppelin_rest_endpoint_tips: '请输入zeppelin server的rest endpoint', + hive_cli_task_execution_type: 'Hive Cli 任务类型', + hive_sql_script: 'Hive SQL 脚本', + hive_cli_options: 'Hive Cli 选项', + hive_cli_options_tips: '请输入您的Hive Cli选项,如--verbose等', jupyter_conda_env_name: 'condaEnvName', jupyter_conda_env_name_tips: '请输入papermill所在的conda环境名', jupyter_input_note_path: 'inputNotePath', diff --git a/dolphinscheduler-ui/src/service/modules/k8s-namespace/types.ts b/dolphinscheduler-ui/src/service/modules/k8s-namespace/types.ts index 7ceae5c86e..203ea0f518 100644 --- a/dolphinscheduler-ui/src/service/modules/k8s-namespace/types.ts +++ b/dolphinscheduler-ui/src/service/modules/k8s-namespace/types.ts @@ -49,4 +49,4 @@ interface NamespaceListRes { start: number } -export { ListReq, K8SReq, NamespaceItem, NamespaceListRes } \ No newline at end of file +export { ListReq, K8SReq, NamespaceItem, NamespaceListRes } diff --git a/dolphinscheduler-ui/src/views/data-quality/rule/index.tsx b/dolphinscheduler-ui/src/views/data-quality/rule/index.tsx index 090d275669..f7d9a60209 100644 --- a/dolphinscheduler-ui/src/views/data-quality/rule/index.tsx +++ b/dolphinscheduler-ui/src/views/data-quality/rule/index.tsx @@ -15,7 +15,13 @@ * limitations under the License. */ -import { defineComponent, getCurrentInstance, onMounted, ref, toRefs } from 'vue' +import { + defineComponent, + getCurrentInstance, + onMounted, + ref, + toRefs +} from 'vue' import { NSpace, NInput, diff --git a/dolphinscheduler-ui/src/views/data-quality/task-result/index.tsx b/dolphinscheduler-ui/src/views/data-quality/task-result/index.tsx index 9a4ba39989..1e07e7c880 100644 --- a/dolphinscheduler-ui/src/views/data-quality/task-result/index.tsx +++ b/dolphinscheduler-ui/src/views/data-quality/task-result/index.tsx @@ -15,7 +15,13 @@ * limitations under the License. */ -import { defineComponent, getCurrentInstance, onMounted, toRefs, watch } from 'vue' +import { + defineComponent, + getCurrentInstance, + onMounted, + toRefs, + watch +} from 'vue' import { NSpace, NInput, @@ -24,7 +30,7 @@ import { NButton, NIcon, NDataTable, - NPagination, + NPagination } from 'naive-ui' import { SearchOutlined } from '@vicons/antd' import { useTable } from './use-table' diff --git a/dolphinscheduler-ui/src/views/data-quality/task-result/use-table.ts b/dolphinscheduler-ui/src/views/data-quality/task-result/use-table.ts index 80cfadc5e0..333171b0b9 100644 --- a/dolphinscheduler-ui/src/views/data-quality/task-result/use-table.ts +++ b/dolphinscheduler-ui/src/views/data-quality/task-result/use-table.ts @@ -30,9 +30,9 @@ import type { ResultListRes } from '@/service/modules/data-quality/types' import { parseTime } from '@/common/common' -import ButtonLink from "@/components/button-link"; -import { NEllipsis, NTag } from "naive-ui"; -import { useRouter } from "vue-router"; +import ButtonLink from '@/components/button-link' +import { NEllipsis, NTag } from 'naive-ui' +import { useRouter } from 'vue-router' export function useTable() { const { t } = useI18n() @@ -76,7 +76,10 @@ export function useTable() { onClick: () => void router.push({ name: 'workflow-instance-detail', - params: { projectCode: row.projectCode, id: row.processInstanceId }, + params: { + projectCode: row.projectCode, + id: row.processInstanceId + }, query: { code: row.processDefinitionCode } }) }, @@ -117,27 +120,27 @@ export function useTable() { render: (row: ResultItem) => { if (row.state === 0) { return h( - NTag, - { type: 'info', size: 'small' }, - { - default: () => t('data_quality.task_result.undone') - } + NTag, + { type: 'info', size: 'small' }, + { + default: () => t('data_quality.task_result.undone') + } ) } else if (row.state === 1) { return h( - NTag, - { type: 'success', size: 'small' }, - { - default: () => t('data_quality.task_result.success') - } + NTag, + { type: 'success', size: 'small' }, + { + default: () => t('data_quality.task_result.success') + } ) } else if (row.state === 2) { return h( - NTag, - { type: 'error', size: 'small' }, - { - default: () => t('data_quality.task_result.failure') - } + NTag, + { type: 'error', size: 'small' }, + { + default: () => t('data_quality.task_result.failure') + } ) } else { return '-' diff --git a/dolphinscheduler-ui/src/views/datasource/list/detail.tsx b/dolphinscheduler-ui/src/views/datasource/list/detail.tsx index dbbfc01988..951ec34e46 100644 --- a/dolphinscheduler-ui/src/views/datasource/list/detail.tsx +++ b/dolphinscheduler-ui/src/views/datasource/list/detail.tsx @@ -15,7 +15,13 @@ * limitations under the License. */ -import { defineComponent, getCurrentInstance, PropType, toRefs, watch } from 'vue' +import { + defineComponent, + getCurrentInstance, + PropType, + toRefs, + watch +} from 'vue' import { NButton, NSpin, @@ -91,10 +97,10 @@ const DetailModal = defineComponent({ async () => { props.show && state.detailForm.type && - await changeType( + (await changeType( state.detailForm.type, datasourceType[state.detailForm.type] - ) + )) props.show && props.id && setFieldsValue(await queryById(props.id)) } ) @@ -171,7 +177,7 @@ const DetailModal = defineComponent({ show-require-mark > {t('datasource.create_datasource')} - +
{item && ( - + )}
diff --git a/dolphinscheduler-ui/src/views/monitor/servers/worker/index.tsx b/dolphinscheduler-ui/src/views/monitor/servers/worker/index.tsx index 2eb682310f..35bddf8dc2 100644 --- a/dolphinscheduler-ui/src/views/monitor/servers/worker/index.tsx +++ b/dolphinscheduler-ui/src/views/monitor/servers/worker/index.tsx @@ -133,11 +133,11 @@ const worker = defineComponent({
{item && ( - + )}
diff --git a/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/index.tsx b/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/index.tsx index 2b8da93fb9..64ae7beff0 100644 --- a/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/index.tsx +++ b/dolphinscheduler-ui/src/views/monitor/statistics/audit-log/index.tsx @@ -15,7 +15,13 @@ * limitations under the License. */ -import { defineComponent, getCurrentInstance, onMounted, toRefs, watch } from 'vue' +import { + defineComponent, + getCurrentInstance, + onMounted, + toRefs, + watch +} from 'vue' import { NSpace, NInput, @@ -85,7 +91,7 @@ const AuditLog = defineComponent({ diff --git a/dolphinscheduler-ui/src/views/projects/list/components/project-modal.tsx b/dolphinscheduler-ui/src/views/projects/list/components/project-modal.tsx index 7e67942ee9..fbf3db9777 100644 --- a/dolphinscheduler-ui/src/views/projects/list/components/project-modal.tsx +++ b/dolphinscheduler-ui/src/views/projects/list/components/project-modal.tsx @@ -15,7 +15,13 @@ * limitations under the License. */ -import { defineComponent, getCurrentInstance, PropType, toRefs, watch } from 'vue' +import { + defineComponent, + getCurrentInstance, + PropType, + toRefs, + watch +} from 'vue' import { NForm, NFormItem, NInput } from 'naive-ui' import { useForm } from './use-form' import Modal from '@/components/modal' @@ -90,7 +96,7 @@ const ProjectModal = defineComponent({ } ) - return { ...toRefs(variables), t, cancelModal, confirmModal, trim} + return { ...toRefs(variables), t, cancelModal, confirmModal, trim } }, render() { const { t } = this @@ -112,7 +118,7 @@ const ProjectModal = defineComponent({ (model.programType === 'RUN_JOB_FLOW' ? 24 : 0)) + const jobFlowDefineJsonSpan = computed(() => + model.programType === 'RUN_JOB_FLOW' ? 24 : 0 + ) - const stepsDefineJsonSpan = computed(() => (model.programType === 'ADD_JOB_FLOW_STEPS' ? 24 : 0)) + const stepsDefineJsonSpan = computed(() => + model.programType === 'ADD_JOB_FLOW_STEPS' ? 24 : 0 + ) return [ { @@ -82,4 +86,4 @@ export const PROGRAM_TYPES = [ label: 'ADD_JOB_FLOW_STEPS', value: 'ADD_JOB_FLOW_STEPS' } -] \ No newline at end of file +] diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-hive-cli.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-hive-cli.ts new file mode 100644 index 0000000000..491a82cf0d --- /dev/null +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-hive-cli.ts @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { useI18n } from 'vue-i18n' +import { useCustomParams, useResources } from '.' +import type { IJsonItem } from '../types' + +export function useHiveCli(model: { [field: string]: any }): IJsonItem[] { + const { t } = useI18n() + + return [ + { + type: 'select', + field: 'hiveCliTaskExecutionType', + span: 12, + name: t('project.node.hive_cli_task_execution_type'), + options: HIVE_CLI_TASK_EXECUTION_TYPES + }, + { + type: 'editor', + field: 'hiveSqlScript', + name: t('project.node.hive_sql_script'), + props: { + language: 'sql' + } + }, + { + type: 'input', + field: 'hiveCliOptions', + name: t('project.node.hive_cli_options'), + props: { + placeholder: t('project.node.hive_cli_options_tips') + } + }, + useResources(), + ...useCustomParams({ model, field: 'localParams', isSimple: false }) + ] +} + +export const HIVE_CLI_TASK_EXECUTION_TYPES = [ + { + label: 'FROM_SCRIPT', + value: 'SCRIPT' + }, + { + label: 'FROM_FILE', + value: 'FILE' + } +] diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-mlflow-models.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-mlflow-models.ts index 622d49f291..9939bbb5aa 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-mlflow-models.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-mlflow-models.ts @@ -47,8 +47,8 @@ export function useMlflowModels(model: { [field: string]: any }): IJsonItem[] { watch( () => [model.deployType], () => { - cpuLimitSpan.value = model.deployType === "DOCKER COMPOSE" ? 12 : 0 - memoryLimitSpan.value = model.deployType === "DOCKER COMPOSE" ? 12 : 0 + cpuLimitSpan.value = model.deployType === 'DOCKER COMPOSE' ? 12 : 0 + memoryLimitSpan.value = model.deployType === 'DOCKER COMPOSE' ? 12 : 0 } ) diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-pytorch.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-pytorch.ts index a74e56f992..7eeebab581 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-pytorch.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-pytorch.ts @@ -46,7 +46,8 @@ export function usePytorch(model: { [field: string]: any }): IJsonItem[] { isCreateEnvironmentSpan.value = model.otherParams ? 12 : 0 pythonPathSpan.value = model.otherParams ? 24 : 0 pythonEnvToolSpan.value = model.showCreateEnvironment ? 12 : 0 - pythonCommandSpan.value = ~model.showCreateEnvironment & model.otherParams ? 12 : 0 + pythonCommandSpan.value = + ~model.showCreateEnvironment & model.otherParams ? 12 : 0 requirementsSpan.value = model.showCreateEnvironment ? 24 : 0 condaPythonVersionSpan.value = model.showCreateConda ? 24 : 0 } diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-resource-limit.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-resource-limit.ts index e5f2ad557c..15c804113e 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-resource-limit.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-resource-limit.ts @@ -29,7 +29,7 @@ export function useResourceLimit(): IJsonItem[] { slots: { suffix: () => '%' }, - props: {min: -1} + props: { min: -1 } }, { type: 'input-number', @@ -39,7 +39,7 @@ export function useResourceLimit(): IJsonItem[] { slots: { suffix: () => t('project.node.mb') }, - props: {min: -1} + props: { min: -1 } } ] } diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sagemaker.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sagemaker.ts index 83e52b874e..ad78353727 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sagemaker.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sagemaker.ts @@ -18,19 +18,18 @@ import type { IJsonItem } from '../types' import { useCustomParams } from '.' export function useSagemaker(model: { [field: string]: any }): IJsonItem[] { - return [ { type: 'editor', field: 'sagemakerRequestJson', - name: "SagemakerRequestJson", + name: 'SagemakerRequestJson', props: { language: 'json' }, validate: { trigger: ['input', 'trigger'], required: true, - message: 'requestJson' + message: 'requestJson' } }, ...useCustomParams({ model, field: 'localParams', isSimple: false }) diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts index becaafea85..0d845fafc1 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts @@ -421,6 +421,12 @@ export function formatParams(data: INodeData): { taskParams.targetJobName = data.targetJobName } + if (data.taskType === 'HIVECLI') { + taskParams.hiveCliTaskExecutionType = data.hiveCliTaskExecutionType + taskParams.hiveSqlScript = data.hiveSqlScript + taskParams.hiveCliOptions = data.hiveCliOptions + } + let timeoutNotifyStrategy = '' if (data.timeoutNotifyStrategy) { if (data.timeoutNotifyStrategy.length === 1) { @@ -658,5 +664,6 @@ export function formatModel(data: ITaskData) { if (data.taskParams?.jobType) { params.isCustomTask = data.taskParams.jobType === 'CUSTOM' } + return params } diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/index.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/index.ts index 0ac47ea9b4..1217de2aeb 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/index.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/index.ts @@ -44,6 +44,7 @@ import { useDinky } from './use-dinky' import { userSagemaker } from './use-sagemaker' import { useChunjun } from './use-chunjun' import { usePytorch } from './use-pytorch' +import { useHiveCli } from './use-hive-cli' export default { SHELL: useShell, @@ -74,5 +75,6 @@ export default { SAGEMAKER: userSagemaker, CHUNJUN: useChunjun, FLINK_STREAM: useFlinkStream, - PYTORCH: usePytorch + PYTORCH: usePytorch, + HIVECLI: useHiveCli } diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-hive-cli.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-hive-cli.ts new file mode 100644 index 0000000000..1c125e656e --- /dev/null +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-hive-cli.ts @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { reactive } from 'vue' +import * as Fields from '../fields/index' +import type { IJsonItem, INodeData, ITaskData } from '../types' + +export function useHiveCli({ + projectCode, + from = 0, + readonly, + data +}: { + projectCode: number + from?: number + readonly?: boolean + data?: ITaskData +}) { + const model = reactive({ + name: '', + taskType: 'HIVECLI', + flag: 'YES', + description: '', + timeoutFlag: false, + localParams: [], + environmentCode: null, + failRetryInterval: 1, + failRetryTimes: 0, + workerGroup: 'default', + delayTime: 0, + timeout: 30 + } as INodeData) + + let extra: IJsonItem[] = [] + if (from === 1) { + extra = [ + Fields.useTaskType(model, readonly), + Fields.useProcessName({ + model, + projectCode, + isCreate: !data?.id, + from, + processName: data?.processName + }) + ] + } + + return { + json: [ + Fields.useName(from), + ...extra, + Fields.useRunFlag(), + Fields.useDescription(), + Fields.useTaskPriority(), + Fields.useWorkerGroup(), + Fields.useEnvironmentName(model, !model.id), + ...Fields.useTaskGroup(model, projectCode), + ...Fields.useFailed(), + Fields.useDelayTime(model), + ...Fields.useTimeoutAlarm(model), + ...Fields.useHiveCli(model), + Fields.usePreTasks() + ] as IJsonItem[], + model + } +} diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-pytorch.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-pytorch.ts index 7b24a07adf..3f3172d419 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-pytorch.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-pytorch.ts @@ -85,4 +85,3 @@ export function usePytorch({ model } } - diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts index 6260be2140..2dba623ea6 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts @@ -306,6 +306,9 @@ interface ITaskParams { restEndpoint?: string zeppelinProductionNoteDirectory?: string productionNoteDirectory?: string + hiveCliOptions?: string + hiveSqlScript?: string + hiveCliTaskExecutionType?: string noteId?: string paragraphId?: string condaEnvName?: string diff --git a/dolphinscheduler-ui/src/views/projects/task/constants/task-type.ts b/dolphinscheduler-ui/src/views/projects/task/constants/task-type.ts index 26505d6a39..0ecc53bccc 100644 --- a/dolphinscheduler-ui/src/views/projects/task/constants/task-type.ts +++ b/dolphinscheduler-ui/src/views/projects/task/constants/task-type.ts @@ -44,6 +44,7 @@ export type TaskType = | 'CHUNJUN' | 'FLINK_STREAM' | 'PYTORCH' + | 'HIVECLI' export type TaskExecuteType = 'STREAM' | 'BATCH' @@ -151,6 +152,10 @@ export const TASK_TYPES_MAP = { PYTORCH: { alias: 'Pytorch', helperLinkDisable: true + }, + HIVECLI: { + alias: 'HIVECLI', + helperLinkDisable: true } } as { [key in TaskType]: { diff --git a/dolphinscheduler-ui/src/views/projects/task/definition/components/start-modal.tsx b/dolphinscheduler-ui/src/views/projects/task/definition/components/start-modal.tsx index b8ae6db145..0834a2d2cc 100644 --- a/dolphinscheduler-ui/src/views/projects/task/definition/components/start-modal.tsx +++ b/dolphinscheduler-ui/src/views/projects/task/definition/components/start-modal.tsx @@ -160,9 +160,7 @@ export default defineComponent({ > -
- {this.row.taskName} -
+
{this.row.taskName}
{t('project.node.start')} - ) - } + + )} {this.menuDisplay && ( <> route.name === 'workflow-definition-detail') + const showTaskDependType = computed( + () => route.name === 'workflow-definition-detail' + ) const renderLabel = (option: any) => { return [ @@ -267,13 +269,19 @@ export default defineComponent({ > - {t('project.workflow.backward_execution')} - {t('project.workflow.forward_execution')} - {t('project.workflow.current_node_execution')} + + {t('project.workflow.backward_execution')} + + + {t('project.workflow.forward_execution')} + + + {t('project.workflow.current_node_execution')} + - ) - } + + )} - + {t('project.workflow.close')} @@ -380,7 +390,7 @@ export default defineComponent({ {t('project.workflow.custom_parallelism')} ) : ( ( ( {t('project.workflow.create_workflow')} - (this.showRef = true)}> + (this.showRef = true)} + > {t('project.workflow.import_workflow')} diff --git a/dolphinscheduler-ui/src/views/projects/workflow/instance/components/process-instance-condition.tsx b/dolphinscheduler-ui/src/views/projects/workflow/instance/components/process-instance-condition.tsx index c1f9efc8f8..29289712d7 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/instance/components/process-instance-condition.tsx +++ b/dolphinscheduler-ui/src/views/projects/workflow/instance/components/process-instance-condition.tsx @@ -16,14 +16,7 @@ */ import { SearchOutlined } from '@vicons/antd' -import { - NInput, - NButton, - NDatePicker, - NSelect, - NIcon, - NSpace -} from 'naive-ui' +import { NInput, NButton, NDatePicker, NSelect, NIcon, NSpace } from 'naive-ui' import { defineComponent, getCurrentInstance, ref } from 'vue' import { useI18n } from 'vue-i18n' import { format } from 'date-fns' @@ -102,7 +95,7 @@ export default defineComponent({ diff --git a/dolphinscheduler-ui/src/views/resource/file/create/index.tsx b/dolphinscheduler-ui/src/views/resource/file/create/index.tsx index 49c2a8ab32..4465714dcf 100644 --- a/dolphinscheduler-ui/src/views/resource/file/create/index.tsx +++ b/dolphinscheduler-ui/src/views/resource/file/create/index.tsx @@ -75,7 +75,7 @@ export default defineComponent({ > + allowInput={this.trim} + v-model:value={this.formData.priority} + />
diff --git a/dolphinscheduler-ui/src/views/resource/task-group/queue/index.tsx b/dolphinscheduler-ui/src/views/resource/task-group/queue/index.tsx index 5891a446f1..ee9233a67a 100644 --- a/dolphinscheduler-ui/src/views/resource/task-group/queue/index.tsx +++ b/dolphinscheduler-ui/src/views/resource/task-group/queue/index.tsx @@ -15,7 +15,14 @@ * limitations under the License. */ -import { ref, defineComponent, toRefs, reactive, onMounted, getCurrentInstance } from 'vue' +import { + ref, + defineComponent, + toRefs, + reactive, + onMounted, + getCurrentInstance +} from 'vue' import { NButton, NIcon, diff --git a/dolphinscheduler-ui/src/views/resource/udf/function/components/function-modal.tsx b/dolphinscheduler-ui/src/views/resource/udf/function/components/function-modal.tsx index 59ddb9dfd4..8f05cbd0ac 100644 --- a/dolphinscheduler-ui/src/views/resource/udf/function/components/function-modal.tsx +++ b/dolphinscheduler-ui/src/views/resource/udf/function/components/function-modal.tsx @@ -15,7 +15,15 @@ * limitations under the License. */ -import { defineComponent, toRefs, PropType, watch, onMounted, ref, getCurrentInstance } from 'vue' +import { + defineComponent, + toRefs, + PropType, + watch, + onMounted, + ref, + getCurrentInstance +} from 'vue' import { NUpload, NIcon, @@ -146,7 +154,7 @@ export default defineComponent({ path='funcName' > @@ -263,7 +271,7 @@ export default defineComponent({ path='description' > - +
diff --git a/dolphinscheduler-ui/src/views/resource/udf/resource/components/upload-modal.tsx b/dolphinscheduler-ui/src/views/resource/udf/resource/components/upload-modal.tsx index 5aacbc6145..ca52b4c593 100644 --- a/dolphinscheduler-ui/src/views/resource/udf/resource/components/upload-modal.tsx +++ b/dolphinscheduler-ui/src/views/resource/udf/resource/components/upload-modal.tsx @@ -86,7 +86,7 @@ export default defineComponent({ )} - + - + props.showModalRef, - () => { - props.showModalRef && getListData() - } - ) - - watch( - () => props.statusRef, - () => { - if (props.statusRef === 0) { - variables.model.namespace = '' - variables.model.clusterCode = '' - variables.model.limitsCpu = '' - variables.model.limitsMemory = '' - variables.model.userId = '' - } else { - variables.model.id = props.row.id - variables.model.namespace = props.row.namespace - variables.model.clusterCode = props.row.clusterCode - variables.model.limitsCpu = props.row.limitsCpu + '' - variables.model.limitsMemory = props.row.limitsMemory + '' - variables.model.userId = props.row.userId - } - } + () => props.showModalRef, + () => { + props.showModalRef && getListData() + } ) watch( - () => props.row, - () => { + () => props.statusRef, + () => { + if (props.statusRef === 0) { + variables.model.namespace = '' + variables.model.clusterCode = '' + variables.model.limitsCpu = '' + variables.model.limitsMemory = '' + variables.model.userId = '' + } else { variables.model.id = props.row.id variables.model.namespace = props.row.namespace variables.model.clusterCode = props.row.clusterCode @@ -103,6 +96,19 @@ const K8sNamespaceModal = defineComponent({ variables.model.limitsMemory = props.row.limitsMemory + '' variables.model.userId = props.row.userId } + } + ) + + watch( + () => props.row, + () => { + variables.model.id = props.row.id + variables.model.namespace = props.row.namespace + variables.model.clusterCode = props.row.clusterCode + variables.model.limitsCpu = props.row.limitsCpu + '' + variables.model.limitsMemory = props.row.limitsMemory + '' + variables.model.userId = props.row.userId + } ) return { t, ...toRefs(variables), cancelModal, confirmModal, trim } @@ -110,83 +116,87 @@ const K8sNamespaceModal = defineComponent({ render() { const { t } = this return ( -
- - {{ - default: () => ( - - - - - - - - - - - CORE - - - - - - GB - - - - ) - }} - -
+
+ + {{ + default: () => ( + + + + + + + + + + + CORE + + + + + + GB + + + + ) + }} + +
) } }) -export default K8sNamespaceModal \ No newline at end of file +export default K8sNamespaceModal diff --git a/dolphinscheduler-ui/src/views/security/k8s-namespace-manage/components/use-modal.ts b/dolphinscheduler-ui/src/views/security/k8s-namespace-manage/components/use-modal.ts index 92248c7175..63c75ba29c 100644 --- a/dolphinscheduler-ui/src/views/security/k8s-namespace-manage/components/use-modal.ts +++ b/dolphinscheduler-ui/src/views/security/k8s-namespace-manage/components/use-modal.ts @@ -26,8 +26,8 @@ import { queryAllClusterList } from '@/service/modules/cluster' import { useAsyncState } from '@vueuse/core' export function useModal( - props: any, - ctx: SetupContext<('cancelModal' | 'confirmModal')[]> + props: any, + ctx: SetupContext<('cancelModal' | 'confirmModal')[]> ) { const { t } = useI18n() @@ -73,8 +73,8 @@ export function useModal( try { statusRef === 0 - ? await submitK8SNamespaceModal() - : await updateK8SNamespaceModal() + ? await submitK8SNamespaceModal() + : await updateK8SNamespaceModal() variables.saving = false } catch (err) { variables.saving = false @@ -83,23 +83,23 @@ export function useModal( const getListData = () => { const { state } = useAsyncState( - queryAllClusterList().then((res: any) => { - variables.model.clusterOptions = res - .filter((item: any) => { - if (item.config) { - const k8s = JSON.parse(item.config).k8s - return !!k8s - } - return false - }) - .map((item: any) => { - return { - label: item.name, - value: item.code - } - }) - }), - {} + queryAllClusterList().then((res: any) => { + variables.model.clusterOptions = res + .filter((item: any) => { + if (item.config) { + const k8s = JSON.parse(item.config).k8s + return !!k8s + } + return false + }) + .map((item: any) => { + return { + label: item.name, + value: item.code + } + }) + }), + {} ) return state @@ -120,9 +120,9 @@ export function useModal( const updateK8SNamespaceModal = () => { updateK8sNamespace(variables.model, variables.model.id).then( - (ignored: any) => { - ctx.emit('confirmModal', props.showModalRef) - } + (ignored: any) => { + ctx.emit('confirmModal', props.showModalRef) + } ) } @@ -131,4 +131,4 @@ export function useModal( handleValidate, getListData } -} \ No newline at end of file +} diff --git a/dolphinscheduler-ui/src/views/security/k8s-namespace-manage/index.tsx b/dolphinscheduler-ui/src/views/security/k8s-namespace-manage/index.tsx index 07eda28cea..e41c79cc27 100644 --- a/dolphinscheduler-ui/src/views/security/k8s-namespace-manage/index.tsx +++ b/dolphinscheduler-ui/src/views/security/k8s-namespace-manage/index.tsx @@ -15,7 +15,13 @@ * limitations under the License. */ -import { defineComponent, getCurrentInstance, onMounted, toRefs, watch } from 'vue' +import { + defineComponent, + getCurrentInstance, + onMounted, + toRefs, + watch +} from 'vue' import { NButton, NDataTable, @@ -112,7 +118,7 @@ const k8sNamespaceManage = defineComponent({ 1 - ? variables.page - 1 - : variables.page, + variables.tableData.length === 1 && variables.page > 1 + ? variables.page - 1 + : variables.page, searchVal: variables.searchVal }) }) @@ -104,61 +104,61 @@ export function useTable() { return h(NSpace, null, { default: () => [ h( - NTooltip, - {}, - { - trigger: () => - h( + NTooltip, + {}, + { + trigger: () => + h( + NButton, + { + circle: true, + type: 'info', + size: 'small', + onClick: () => { + handleEdit(row) + } + }, + { + icon: () => + h(NIcon, null, { default: () => h(EditOutlined) }) + } + ), + default: () => t('security.k8s_namespace.edit') + } + ), + h( + NPopconfirm, + { + onPositiveClick: () => { + handleDelete(row) + } + }, + { + trigger: () => + h( + NTooltip, + {}, + { + trigger: () => + h( NButton, { circle: true, - type: 'info', - size: 'small', - onClick: () => { - handleEdit(row) - } + type: 'error', + size: 'small' }, { icon: () => - h(NIcon, null, { default: () => h(EditOutlined) }) + h(NIcon, null, { + default: () => h(DeleteOutlined) + }) } - ), - default: () => t('security.k8s_namespace.edit') - } - ), - h( - NPopconfirm, - { - onPositiveClick: () => { - handleDelete(row) - } - }, - { - trigger: () => - h( - NTooltip, - {}, - { - trigger: () => - h( - NButton, - { - circle: true, - type: 'error', - size: 'small' - }, - { - icon: () => - h(NIcon, null, { - default: () => h(DeleteOutlined) - }) - } - ), - default: () => t('security.k8s_namespace.delete') - } - ), - default: () => t('security.k8s_namespace.delete_confirm') - } + ), + default: () => t('security.k8s_namespace.delete') + } + ), + default: () => t('security.k8s_namespace.delete_confirm') + } ) ] }) @@ -188,24 +188,24 @@ export function useTable() { if (variables.loadingRef) return variables.loadingRef = true const { state } = useAsyncState( - queryNamespaceListPaging({ ...params }).then((res: NamespaceListRes) => { - variables.tableData = res.totalList.map((item, unused) => { - item.createTime = format( - parseTime(item.createTime), - 'yyyy-MM-dd HH:mm:ss' - ) - item.updateTime = format( - parseTime(item.updateTime), - 'yyyy-MM-dd HH:mm:ss' - ) - return { - ...item - } - }) as any - variables.totalPage = res.totalPage - variables.loadingRef = false - }), - {} + queryNamespaceListPaging({ ...params }).then((res: NamespaceListRes) => { + variables.tableData = res.totalList.map((item, unused) => { + item.createTime = format( + parseTime(item.createTime), + 'yyyy-MM-dd HH:mm:ss' + ) + item.updateTime = format( + parseTime(item.updateTime), + 'yyyy-MM-dd HH:mm:ss' + ) + return { + ...item + } + }) as any + variables.totalPage = res.totalPage + variables.loadingRef = false + }), + {} ) return state @@ -216,4 +216,4 @@ export function useTable() { getTableData, createColumns } -} \ No newline at end of file +} diff --git a/dolphinscheduler-ui/src/views/security/tenant-manage/components/tenant-modal.tsx b/dolphinscheduler-ui/src/views/security/tenant-manage/components/tenant-modal.tsx index fd3086d843..ffdc73534f 100644 --- a/dolphinscheduler-ui/src/views/security/tenant-manage/components/tenant-modal.tsx +++ b/dolphinscheduler-ui/src/views/security/tenant-manage/components/tenant-modal.tsx @@ -15,7 +15,13 @@ * limitations under the License. */ -import { defineComponent, getCurrentInstance, PropType, toRefs, watch } from 'vue' +import { + defineComponent, + getCurrentInstance, + PropType, + toRefs, + watch +} from 'vue' import Modal from '@/components/modal' import { NForm, NFormItem, NInput, NSelect } from 'naive-ui' import { useModalData } from './use-modalData' @@ -123,7 +129,7 @@ const TenantModal = defineComponent({ path='tenantCode' > + allowInput={this.trim} + v-model:value={this.searchVal} + size='small' + clearable + /> diff --git a/dolphinscheduler-ui/src/views/security/worker-group-manage/components/worker-group-modal.tsx b/dolphinscheduler-ui/src/views/security/worker-group-manage/components/worker-group-modal.tsx index b84f0ba84b..156810e21d 100644 --- a/dolphinscheduler-ui/src/views/security/worker-group-manage/components/worker-group-modal.tsx +++ b/dolphinscheduler-ui/src/views/security/worker-group-manage/components/worker-group-modal.tsx @@ -15,7 +15,13 @@ * limitations under the License. */ -import { defineComponent, getCurrentInstance, PropType, toRefs, watch } from 'vue' +import { + defineComponent, + getCurrentInstance, + PropType, + toRefs, + watch +} from 'vue' import Modal from '@/components/modal' import { NForm, NFormItem, NInput, NSelect } from 'naive-ui' import { useModal } from './use-modal' @@ -118,7 +124,7 @@ const WorkerGroupModal = defineComponent({ path='name' >