diff --git a/docs/configs/docsdev.js b/docs/configs/docsdev.js index d7cac77fa9..3dfd9b3166 100644 --- a/docs/configs/docsdev.js +++ b/docs/configs/docsdev.js @@ -185,6 +185,10 @@ export default { title: 'Dinky', link: '/en-us/docs/dev/user_doc/guide/task/dinky.html', }, + { + title: 'Java', + link: '/en-us/docs/dev/user_doc/guide/task/java.html', + }, { title: 'SageMaker', link: '/en-us/docs/dev/user_doc/guide/task/sagemaker.html', @@ -809,6 +813,10 @@ export default { title: 'Dinky', link: '/zh-cn/docs/dev/user_doc/guide/task/dinky.html', }, + { + title: 'Java', + link: '/zh-cn/docs/dev/user_doc/guide/task/java.html', + }, { title: 'SageMaker', link: '/zh-cn/docs/dev/user_doc/guide/task/sagemaker.html', diff --git a/docs/docs/en/guide/task/java.md b/docs/docs/en/guide/task/java.md new file mode 100644 index 0000000000..95bf5c28f9 --- /dev/null +++ b/docs/docs/en/guide/task/java.md @@ -0,0 +1,47 @@ +# Overview + +This node is for executing java-type tasks and supports using files and jar packages as program entries. + +# Create Tasks + +- Click on `Project Management` -> `Project Name` -> `Workflow Definition`, click on the “Create workflow” button, go to the DAG edit page: + +- Drag the toolbar's Java task node to the palette. + +# Task Parameters +| **Parameter** | **Description** | +|--------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Node Name | The name of the set task. The node name in a workflow definition is unique. | +| Run Flag | Indicates whether the node is scheduled properly and turns on the kill switch, if not needed. | +| Description | Describes the functionality of the node. | +| Task Priority | When the number of worker threads is insufficient, the worker executes tasks according to the priority. When the priority is the same, the worker executes tasks by order. | +| Worker Group | The group of machines who execute the tasks. If selecting `Default`, DolphinScheduler will randomly choose a worker machine to execute the task. | +| Environment Name | Configure the environment in which the task runs. | +| Number Of Failed Retries | Number of resubmitted tasks that failed. You can choose the number in the drop-down menu or fill it manually. | +| Failed Retry Interval | the interval between the failure and resubmission of a task. You can choose the number in the drop-down menu or fill it manually. | +| Delayed Execution Time | the amount of time a task is delayed, in units. | +| Timeout Alarm | Check timeout warning, timeout failure, when the task exceeds the“Timeout length”, send a warning message and the task execution fails. | +| Module Path | pick Java 9 + 's modularity feature, put all resources into-module-path, and require that the JDK version in your worker supports modularity. | +| Main Parameter | Java program main method entry parameter. | +| Java VM Parameters | JVM startup parameters. | +| Script | You need to write Java code if you use the Java run type. The public class must exist in the code without writing a package statement. | +| Resources | External JAR packages or other resource files that are added to the classpath or module path and can be easily retrieved in your JAVA script. | +| Custom parameter | A user-defined parameter that is part of HTTP and replaces `${ variable }` in the script . | +| Pre Tasks | Selects a pre-task for the current task and sets the pre-task as the upstream of the current task. | + +## Example + +Java type tasks have two modes of execution, here is a demonstration of executing tasks in Java mode. + +The main configuration parameters are as follows: +- Run Type +- Module Path +- Main Parameters +- Java VM Parameters +- Script + +![java_task](../../../../img/tasks/demo/java_task02.png) + +## Note + +When you run the task in JAVA execution mode, the public class must exist in the code, and you could omit writing a package statement. diff --git a/docs/docs/zh/guide/task/java.md b/docs/docs/zh/guide/task/java.md new file mode 100644 index 0000000000..852886dae4 --- /dev/null +++ b/docs/docs/zh/guide/task/java.md @@ -0,0 +1,50 @@ +# JAVA 节点 + +## 综述 + +该节点用于执行 java 类型的任务，支持使用单文件和jar包作为程序入口。 + +## 创建任务 + +- 点击项目管理 -> 项目名称 -> 工作流定义，点击”创建工作流”按钮，进入 DAG 编辑页面： + +- 拖动工具栏的JAVA任务节点到画板中。 + +## 任务参数 + +- 节点名称：设置任务的名称。一个工作流定义中的节点名称是唯一的。 +- 运行标志：标识这个节点是否能正常调度,如果不需要执行，可以打开禁止执行开关。 +- 描述：描述该节点的功能。 +- 任务优先级：worker 线程数不足时，根据优先级从高到低依次执行，优先级一样时根据先进先出原则执行。 +- Worker 分组：任务分配给 worker 组的机器机执行，选择 Default，会随机选择一台 worker 机执行。 +- 环境名称：配置运行任务的环境。 +- 失败重试次数：任务失败重新提交的次数，支持下拉和手填。 +- 失败重试间隔：任务失败重新提交任务的时间间隔，支持下拉和手填。 +- 延迟执行时间：任务延迟执行的时间，以分为单位。 +- 超时告警：勾选超时告警、超时失败，当任务超过"超时时长"后，会发送告警邮件并且任务执行失败。 +- 模块路径：开启使用JAVA9+的模块化特性，把所有资源放入--module-path中，要求您的worker中的JDK版本支持模块化。 +- 主程序参数：作为普通Java程序main方法入口参数。 +- 虚拟机参数：配置启动虚拟机参数。 +- 脚本：若使用JAVA运行类型则需要编写JAVA代码。代码中必须存在public类，不用写package语句。 +- 资源：可以是外部JAR包也可以是其他资源文件，它们都会被加入到类路径或模块路径中，您可以在自己的JAVA脚本中轻松获取。 +- 自定义参数：是 http 局部的用户自定义参数，会替换脚本中以 ${变量} 的内容。 +- 前置任务：选择当前任务的前置任务，会将被选择的前置任务设置为当前任务的上游。 + +## 任务样例 + +java任务类型有两种运行模式，这里以JAVA模式为例进行演示。 + +主要配置参数如下： + +- 运行类型 +- 模块路径 +- 主程序参数 +- 虚拟机参数 +- 脚本文件 + +![java_task](../../../../img/tasks/demo/java_task02.png) + +## 注意事项 + +使用JAVA运行类型时代码中必须存在public类，可以不写package语句 + diff --git a/docs/img/tasks/demo/java_task02.png b/docs/img/tasks/demo/java_task02.png new file mode 100644 index 0000000000..28cc4beb28 Binary files /dev/null and b/docs/img/tasks/demo/java_task02.png differ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml index 665d28783d..ff252760ab 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml @@ -189,6 +189,12 @@ ${project.version} + + org.apache.dolphinscheduler + dolphinscheduler-task-java + ${project.version} + + org.apache.dolphinscheduler dolphinscheduler-task-sagemaker diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/TaskResponse.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/TaskResponse.java index e4b60c1928..241f308dea 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/TaskResponse.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/TaskResponse.java @@ -19,6 +19,9 @@ package org.apache.dolphinscheduler.plugin.task.api.model; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskRunStatus; +import lombok.Data; + +@Data public class TaskResponse { /** @@ -46,8 +49,6 @@ public class TaskResponse { */ private Process process; - - /** * cancel */ @@ -59,68 +60,4 @@ public class TaskResponse { private volatile int exitStatusCode = -1; private TaskRunStatus status; - - public String getVarPool() { - return varPool; - } - - public void setVarPool(String varPool) { - this.varPool = varPool; - } - - public int getProcessId() { - return processId; - } - - public void setProcessId(int processId) { - this.processId = processId; - } - - public String getResultString() { - return resultString; - } - - public void setResultString(String resultString) { - this.resultString = resultString; - } - - public String getAppIds() { - return appIds; - } - - public void setAppIds(String appIds) { - this.appIds = appIds; - } - - public boolean isCancel() { - return cancel; - } - - public void setCancel(boolean cancel) { - this.cancel = cancel; - } - - public int getExitStatusCode() { - return exitStatusCode; - } - - public void setExitStatusCode(int exitStatusCode) { - this.exitStatusCode = exitStatusCode; - } - - public Process getProcess() { - return process; - } - - public void setProcess(Process process) { - this.process = process; - } - - public TaskRunStatus getStatus() { - return status; - } - - public void setStatus(TaskRunStatus status) { - this.status = status; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/pom.xml new file mode 100644 index 0000000000..36aa9afeb2 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/pom.xml @@ -0,0 +1,45 @@ + + + + + dolphinscheduler-task-plugin + org.apache.dolphinscheduler + dev-SNAPSHOT + + + 4.0.0 + dolphinscheduler-task-java + jar + + + + + org.apache.dolphinscheduler + dolphinscheduler-spi + + + + org.apache.dolphinscheduler + dolphinscheduler-task-api + + + + + \ No newline at end of file diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaConstants.java new file mode 100644 index 0000000000..e4d941db30 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaConstants.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.java; + +import java.io.File; + +public class JavaConstants { + + private JavaConstants() { + throw new IllegalStateException("Utility class"); + } + + /** + * The constants used to get the Java installation directory + **/ + public static final String JAVA_HOME_VAR = "${JAVA_HOME}"; + + /** + * this constant represents the use of the java command to run a task + **/ + public static final String RUN_TYPE_JAVA = "JAVA"; + + /** + * this constant represents the use of the java -jar command to run a task + **/ + public static final String RUN_TYPE_JAR = "JAR"; + + /** + * This constant is the Classpath or module path delimiter for different operating systems + **/ + public static final String PATH_SEPARATOR = System.getProperty("path.separator"); + + /** + * This constant represents the current directory in the Classpath or module path + **/ + public static final String CLASSPATH_CURRENT_DIR = "."; + + /** + * This constant is used to construct the pre-pathname of the Java source file + **/ + public static final String JAVA_SOURCE_CODE_NAME_TEMPLATE = "%s/%s.java"; + + /** + * This constant is the regular expression to get the class name of the source file + **/ + public static final String PUBLIC_CLASS_NAME_REGEX = "(.*\\s*public\\s+class\\s+)([a-zA-Z_]+[//w_]*)([.\\s\\S]*)"; +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaParameters.java new file mode 100644 index 0000000000..527441b634 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaParameters.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.java; + +import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import java.util.List; + +import lombok.Data; + +@Data +public class JavaParameters extends AbstractParameters { + /** + * origin java script + */ + private String rawScript; + + /** + * run in jar file + */ + private ResourceInfo mainJar; + + /** + * Marks the current task running mode + */ + private String runType; + + /** + * main method args + **/ + private String mainArgs; + + /** + * java virtual machine args + **/ + private String jvmArgs; + + /** + * module path or class path flag + **/ + private boolean isModulePath; + + /** + * resource list + */ + private List resourceList; + + /** + * Check that the parameters are valid + * + * @returnboolean + */ + @Override + public boolean checkParameters() { + return runType != null && (StringUtils.isNotBlank(rawScript) || mainJar != null); + } + + /** + * Gets a list of known resource files + * + * @return List + **/ + @Override + public List getResourceFilesList() { + return this.resourceList; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java new file mode 100644 index 0000000000..e23eb5124c --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java @@ -0,0 +1,388 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.java; + +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.SINGLE_SLASH; +import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.JAVA_HOME_VAR; +import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.PUBLIC_CLASS_NAME_REGEX; + +import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; +import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor; +import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; +import org.apache.dolphinscheduler.plugin.task.api.TaskException; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; +import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; +import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; +import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; +import org.apache.dolphinscheduler.plugin.task.java.exception.JavaSourceFileExistException; +import org.apache.dolphinscheduler.plugin.task.java.exception.PublicClassNotFoundException; +import org.apache.dolphinscheduler.plugin.task.java.exception.RunTypeNotFoundException; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; + +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.google.common.base.Preconditions; + +public class JavaTask extends AbstractTask { + + /** + * Contains various parameters for this task + */ + private JavaParameters javaParameters; + + /** + * To run shell commands + */ + private ShellCommandExecutor shellCommandExecutor; + + /** + * task execution context + */ + private TaskExecutionContext taskRequest; + + /** + * class name regex pattern + */ + private static final Pattern classNamePattern = Pattern.compile(PUBLIC_CLASS_NAME_REGEX); + + public JavaTask(TaskExecutionContext taskRequest) { + super(taskRequest); + this.taskRequest = taskRequest; + this.shellCommandExecutor = new ShellCommandExecutor(this::logHandle, + taskRequest, + logger); + } + + /** + * Initializes a Java task + * @return void + **/ + @Override + public void init() { + logger.info("java task params {}", taskRequest.getTaskParams()); + javaParameters = JSONUtils.parseObject(taskRequest.getTaskParams(), JavaParameters.class); + if (javaParameters == null || !javaParameters.checkParameters()) { + throw new TaskException("java task params is not valid"); + } + if (javaParameters.getRunType().equals(JavaConstants.RUN_TYPE_JAR)) { + setMainJarName(); + } + } + + /** + * Gets the Java source file that was initially processed + * + * @return String + **/ + @Override + public String getPreScript() { + String rawJavaScript = javaParameters.getRawScript().replaceAll("\\r\\n", "\n"); + try { + rawJavaScript = convertJavaSourceCodePlaceholders(rawJavaScript); + } catch (StringIndexOutOfBoundsException e) { + logger.error("setShareVar field format error, raw java script: {}", rawJavaScript); + } + return rawJavaScript; + } + + /** + * Execute Java tasks + * + * @return void + * @throws Exception + */ + @Override + public void handle(TaskCallBack taskCallBack) throws TaskException { + try { + // Step 1: judge if is java or jar run type. + // Step 2 case1: the jar run type builds the command directly, adding resource to the java -jar class when building the command + // Step 2 case2: the java run type, first replace the custom parameters, then compile the code, and then build the command will add resource + // Step 3: to run the command + String command = null; + switch (javaParameters.getRunType()) { + case JavaConstants.RUN_TYPE_JAVA: + command = buildJavaCommand(); + break; + case JavaConstants.RUN_TYPE_JAR: + command = buildJarCommand(); + break; + default: + throw new RunTypeNotFoundException("run type is required, but it is null now."); + } + Preconditions.checkNotNull(command, "command not be null."); + TaskResponse taskResponse = shellCommandExecutor.run(command); + logger.info("java task run result: {}", taskResponse); + setExitStatusCode(taskResponse.getExitStatusCode()); + setAppIds(taskResponse.getAppIds()); + setProcessId(taskResponse.getProcessId()); + setVarPool(shellCommandExecutor.getVarPool()); + } catch (InterruptedException e) { + logger.error("java task interrupted ", e); + setExitStatusCode(TaskConstants.EXIT_CODE_FAILURE); + Thread.currentThread().interrupt(); + } catch (RunTypeNotFoundException e) { + logger.error(e.getMessage()); + setExitStatusCode(TaskConstants.EXIT_CODE_FAILURE); + throw e; + } catch (Exception e) { + logger.error("java task failed ", e); + setExitStatusCode(TaskConstants.EXIT_CODE_FAILURE); + throw new TaskException("run java task error", e); + } + } + + /** + * Construct a shell command for the java Run mode + * + * @return String + * @throws Exception + **/ + protected String buildJavaCommand() throws Exception { + StringBuilder builder = new StringBuilder(); + String sourceCode = buildJavaSourceContent(); + builder.append(buildJavaCompileCommand(sourceCode)) + .append(";") + .append(getJavaCommandPath()) + .append("java").append(" ") + .append(buildResourcePath()) + .append(" ") + .append(getPublicClassName(sourceCode)) + .append(" ") + .append(javaParameters.getMainArgs().trim()).append(" ") + .append(javaParameters.getJvmArgs().trim()); + return builder.toString(); + } + + private void setMainJarName() { + ResourceInfo mainJar = javaParameters.getMainJar(); + String resourceName = getResourceNameOfMainJar(mainJar); + mainJar.setRes(resourceName); + javaParameters.setMainJar(mainJar); + } + + /** + * Construct a shell command for the java -jar Run mode + * + * @return String + **/ + protected String buildJarCommand() { + String fullName = javaParameters.getMainJar().getResourceName(); + String mainJarName = fullName.substring(0, fullName.lastIndexOf('.')); + mainJarName = mainJarName.substring(mainJarName.lastIndexOf('.') + 1) + ".jar"; + StringBuilder builder = new StringBuilder(); + builder.append(getJavaCommandPath()) + .append("java").append(" ") + .append(buildResourcePath()).append(" ") + .append("-jar").append(" ") + .append(taskRequest.getExecutePath()) + .append(mainJarName).append(" ") + .append(javaParameters.getMainArgs().trim()).append(" ") + .append(javaParameters.getJvmArgs().trim()); + return builder.toString(); + } + + private String getResourceNameOfMainJar(ResourceInfo mainJar) { + if (null == mainJar) { + throw new RuntimeException("The jar for the task is required."); + } + return mainJar.getId() == 0 + ? mainJar.getRes() + // when update resource maybe has error + : mainJar.getResourceName().replaceFirst(SINGLE_SLASH, ""); + } + + @Override + public void cancel() throws TaskException { + // cancel process + try { + shellCommandExecutor.cancelApplication(); + } catch (Exception e) { + throw new TaskException(); + } + } + + @Override + public AbstractParameters getParameters() { + return javaParameters; + } + + + /** + * Replaces placeholders such as local variables in source files + * + * @param rawScript + * @return String + * @throws StringIndexOutOfBoundsException + */ + protected static String convertJavaSourceCodePlaceholders(String rawScript) throws StringIndexOutOfBoundsException { + int len = "${setShareVar(${".length(); + + int scriptStart = 0; + while ((scriptStart = rawScript.indexOf("${setShareVar(${", scriptStart)) != -1) { + int start = -1; + int end = rawScript.indexOf('}', scriptStart + len); + String prop = rawScript.substring(scriptStart + len, end); + + start = rawScript.indexOf(',', end); + end = rawScript.indexOf(')', start); + + String value = rawScript.substring(start + 1, end); + + start = rawScript.indexOf('}', start) + 1; + end = rawScript.length(); + + String replaceScript = String.format("print(\"${{setValue({},{})}}\".format(\"%s\",%s))", prop, value); + + rawScript = rawScript.substring(0, scriptStart) + replaceScript + rawScript.substring(start, end); + + scriptStart += replaceScript.length(); + } + return rawScript; + } + + /** + * Creates a Java source file when it does not exist + * + * @param sourceCode + * @param fileName + * @return String + **/ + protected void createJavaSourceFileIfNotExists(String sourceCode, String fileName) throws IOException { + logger.info("tenantCode: {}, task dir:{}", taskRequest.getTenantCode(), taskRequest.getExecutePath()); + if (!Files.exists(Paths.get(fileName))) { + logger.info("the java source code:{}, will be write to the file: {}", fileName,sourceCode); + // write data to file + FileUtils.writeStringToFile(new File(fileName), + sourceCode, + StandardCharsets.UTF_8); + } else { + throw new JavaSourceFileExistException("java source file exists, please report an issue on official."); + } + } + + /** + * Construct the full path name of the Java source file from the temporary execution path of the task + * + * @return String + **/ + protected String buildJavaSourceCodeFileFullName(String publicClassName) { + return String.format(JavaConstants.JAVA_SOURCE_CODE_NAME_TEMPLATE, taskRequest.getExecutePath(), publicClassName); + } + + /** + * Construct a Classpath or module path based on isModulePath + * + * @return String + **/ + protected String buildResourcePath() { + StringBuilder builder = new StringBuilder(); + if (javaParameters.isModulePath()) { + builder.append("--module-path"); + } else { + builder.append("--class-path"); + } + builder.append(" ").append(JavaConstants.CLASSPATH_CURRENT_DIR) + .append(JavaConstants.PATH_SEPARATOR) + .append(taskRequest.getExecutePath()); + for (ResourceInfo info : javaParameters.getResourceFilesList()) { + builder.append(JavaConstants.PATH_SEPARATOR); + builder.append(taskRequest.getExecutePath()) + .append(info.getResourceName()); + } + return builder.toString(); + } + + /** + * Constructs a shell command compiled from a Java source file + * + * @param sourceCode + * @return String + * @throws IOException + **/ + protected String buildJavaCompileCommand(String sourceCode) throws IOException { + String publicClassName = getPublicClassName(sourceCode); + String fileName = buildJavaSourceCodeFileFullName(publicClassName); + createJavaSourceFileIfNotExists(sourceCode, fileName); + + StringBuilder compilerCommand = new StringBuilder() + .append(getJavaCommandPath()) + .append("javac").append(" ") + .append(buildResourcePath()).append(" ") + .append(fileName); + return compilerCommand.toString(); + } + + /** + * Work with Java source file content, such as replacing local variables + * + * @return String + **/ + protected String buildJavaSourceContent() { + String rawJavaScript = javaParameters.getRawScript().replaceAll("\\r\\n", "\n"); + // replace placeholder + + Map paramsMap = taskRequest.getPrepareParamsMap(); + if (MapUtils.isEmpty(paramsMap)) { + paramsMap = new HashMap<>(); + } + if (MapUtils.isNotEmpty(taskRequest.getParamsMap())) { + paramsMap.putAll(taskRequest.getParamsMap()); + } + logger.info("The current java source code will begin to replace the placeholder: {}", rawJavaScript); + rawJavaScript = ParameterUtils.convertParameterPlaceholders(rawJavaScript, ParamUtils.convert(paramsMap)); + return rawJavaScript; + } + + /** + * Gets the operating system absolute path to the Java command + * + * @return String + **/ + private String getJavaCommandPath() { + return JAVA_HOME_VAR + File.separator + "bin" + File.separator; + } + + /** + * Gets the public class name from the Java source file + * + * @param sourceCode + * @return String + **/ + public String getPublicClassName(String sourceCode) { + Matcher matcher = classNamePattern.matcher(sourceCode); + if (!matcher.find()) { + throw new PublicClassNotFoundException("public class is not be found in source code : " + sourceCode); + } + return matcher.group(2).trim(); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannel.java new file mode 100644 index 0000000000..32407b204a --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannel.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.java; + +import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; +import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; + +public class JavaTaskChannel implements TaskChannel { + + /** + * Cancel the mission + * + * @param status + * @return void + **/ + @Override + public void cancelApplication(boolean status) { + + } + + /** + * Create a task + * + * @param taskRequest This parameter is the Echternach of the mission + * @return JavaTask + **/ + @Override + public JavaTask createTask(TaskExecutionContext taskRequest) { + return new JavaTask(taskRequest); + } + + /** + * Parses Java task parameters + * + * @param parametersNode + * @return: org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters + **/ + @Override + public AbstractParameters parseParameters(ParametersNode parametersNode) { + return JSONUtils.parseObject(parametersNode.getTaskParams(), JavaParameters.class); + } + + /** + * Gets a list of the resources that the task depends on + * + * @param parameters + * @return ResourceParametersHelper + **/ + @Override + public ResourceParametersHelper getResources(String parameters) { + return null; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannelFactory.java new file mode 100644 index 0000000000..c374960d45 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannelFactory.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.java; + +import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; +import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; +import org.apache.dolphinscheduler.spi.params.base.PluginParams; + +import java.util.List; + +import com.google.auto.service.AutoService; + +@AutoService(TaskChannelFactory.class) +public class JavaTaskChannelFactory implements TaskChannelFactory { + /** + * Construct a channel for a Java task + * + * @return TaskChannel + **/ + @Override + public TaskChannel create() { + return new JavaTaskChannel(); + } + + /** + * Get a unique identifier of the Java task + * + * @return String + **/ + @Override + public String getName() { + return "JAVA"; + } + + /** + * Gets the plug-in parameters for the Java task + * + * @return List + **/ + @Override + public List getParams() { + return null; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/JavaSourceFileExistException.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/JavaSourceFileExistException.java new file mode 100644 index 0000000000..421d77de66 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/JavaSourceFileExistException.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.java.exception; + +public class JavaSourceFileExistException extends RuntimeException { + public JavaSourceFileExistException() { + } + + public JavaSourceFileExistException(String message) { + super(message); + } + + public JavaSourceFileExistException(String message, Throwable cause) { + super(message, cause); + } + + public JavaSourceFileExistException(Throwable cause) { + super(cause); + } + + public JavaSourceFileExistException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/PublicClassNotFoundException.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/PublicClassNotFoundException.java new file mode 100644 index 0000000000..16b0070c97 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/PublicClassNotFoundException.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.java.exception; + +public class PublicClassNotFoundException extends RuntimeException { + public PublicClassNotFoundException() { + } + + public PublicClassNotFoundException(String message) { + super(message); + } + + public PublicClassNotFoundException(String message, Throwable cause) { + super(message, cause); + } + + public PublicClassNotFoundException(Throwable cause) { + super(cause); + } + + public PublicClassNotFoundException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/RunTypeNotFoundException.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/RunTypeNotFoundException.java new file mode 100644 index 0000000000..d893690673 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/RunTypeNotFoundException.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.java.exception; + +public class RunTypeNotFoundException extends RuntimeException { + + public RunTypeNotFoundException() { + super(); + } + + public RunTypeNotFoundException(String s) { + super(s); + } + + public RunTypeNotFoundException(String s, Throwable throwable) { + super(s, throwable); + } + + public RunTypeNotFoundException(Throwable throwable) { + super(throwable); + } + + protected RunTypeNotFoundException(String s, Throwable throwable, boolean b, boolean b1) { + super(s, throwable, b, b1); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/test/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/test/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java new file mode 100644 index 0000000000..89cc4f89e5 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/test/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.java; + +import static org.apache.dolphinscheduler.plugin.task.api.enums.DataType.VARCHAR; +import static org.apache.dolphinscheduler.plugin.task.api.enums.Direct.IN; +import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_JAR; +import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_JAVA; + +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; +import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; +import org.apache.dolphinscheduler.plugin.task.java.exception.JavaSourceFileExistException; +import org.apache.dolphinscheduler.plugin.task.java.exception.PublicClassNotFoundException; +import org.apache.dolphinscheduler.plugin.task.java.exception.RunTypeNotFoundException; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; + +import org.junit.Assert; +import org.junit.Test; + +public class JavaTaskTest { + + @Test + public void testGetPubllicClassName(){ + JavaTask javaTask = runJavaType(); + Assert.assertEquals(javaTask.getPublicClassName("import java.io.IOException;\n" + + "public class JavaTaskTest {\n" + + " public static void main(String[] args) throws IOException {\n" + + " StringBuilder builder = new StringBuilder(\"Hello: \");\n" + + " for (String arg : args) {\n" + + " builder.append(arg).append(\" \");\n" + + " }\n" + + " System.out.println(builder);\n" + + " }\n" + + "}\n"), "JavaTaskTest"); + } + + /** + * Construct a java -jar command + * + * @return void + **/ + @Test + public void buildJarCommand() { + String homeBinPath = JavaConstants.JAVA_HOME_VAR + File.separator + "bin" + File.separator; + JavaTask javaTask = runJarType(); + Assert.assertEquals(javaTask.buildJarCommand(), homeBinPath + + "java --class-path .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar -jar /tmp/dolphinscheduler/test/executepath/opt/share/jar/main.jar -host 127.0.0.1 -port 8080 -xms:50m"); + } + + /** + * Construct the compile command + * + * @return void + **/ + @Test + public void buildJavaCompileCommand() throws IOException { + JavaTask javaTask = runJavaType(); + String sourceCode = javaTask.buildJavaSourceContent(); + String publicClassName = javaTask.getPublicClassName(sourceCode); + Assert.assertEquals("JavaTaskTest", publicClassName); + String fileName = javaTask.buildJavaSourceCodeFileFullName(publicClassName); + try { + String homeBinPath = JavaConstants.JAVA_HOME_VAR + File.separator + "bin" + File.separator; + Path path = Paths.get(fileName); + if (Files.exists(path)) { + Files.delete(path); + } + Assert.assertEquals(homeBinPath + + "javac --class-path .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java", + javaTask.buildJavaCompileCommand(sourceCode)); + } finally { + Path path = Paths.get(fileName); + if (Files.exists(path)) { + Files.delete(path); + } + } + + } + + /** + * Construct java to run the command + * + * @return void + **/ + @Test + public void buildJavaCommand() throws Exception { + String wantJavaCommand = "${JAVA_HOME}/bin/javac --class-path .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java;${JAVA_HOME}/bin/java --class-path .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar JavaTaskTest -host 127.0.0.1 -port 8080 -xms:50m"; + JavaTask javaTask = runJavaType(); + String sourceCode = javaTask.buildJavaSourceContent(); + String publicClassName = javaTask.getPublicClassName(sourceCode); + Assert.assertEquals("JavaTaskTest", publicClassName); + String fileName = javaTask.buildJavaSourceCodeFileFullName(publicClassName); + Path path = Paths.get(fileName); + if (Files.exists(path)) { + Files.delete(path); + } + Assert.assertEquals(wantJavaCommand, javaTask.buildJavaCommand()); + } + + /** + * There is no exception to overwriting the Java source file + * @return void + * @throws IOException + **/ + @Test(expected = JavaSourceFileExistException.class) + public void coverJavaSourceFileExistException() throws IOException { + JavaTask javaTask = runJavaType(); + String sourceCode = javaTask.buildJavaSourceContent(); + String publicClassName = javaTask.getPublicClassName(sourceCode); + Assert.assertEquals("JavaTaskTest", publicClassName); + String fileName = javaTask.buildJavaSourceCodeFileFullName(publicClassName); + try { + Path path = Paths.get(fileName); + if (!Files.exists(path)) { + Files.createDirectories(path); + } + javaTask.createJavaSourceFileIfNotExists(sourceCode,fileName); + } finally { + Path path = Paths.get(fileName); + if (Files.exists(path)) { + Files.delete(path); + } + } + } + + /** + * The override class name could not find an exception + * + * @return void + **/ + @Test(expected = PublicClassNotFoundException.class) + public void coverPublicClassNotFoundException() { + JavaTask javaTask = runJavaType(); + javaTask.getPublicClassName(""); + } + + /** + * The override run mode could not find an exception + * + * @return void + * @throws Exception + **/ + @Test(expected = RunTypeNotFoundException.class) + public void coverRunTypeNotFoundException() throws Exception { + JavaTask javaTask = runJavaType(); + Field javaParameters = JavaTask.class.getDeclaredField("javaParameters"); + javaParameters.setAccessible(true); + ((JavaParameters)(javaParameters.get(javaTask))).setRunType(""); + javaTask.handle(); + javaTask.getPublicClassName(""); + } + + /** + * Create a Java task parameter mock object + * + * @param runType + * @return JavaParameters + **/ + public JavaParameters createJavaParametersObject(String runType) { + JavaParameters javaParameters = new JavaParameters(); + javaParameters.setRunType(runType); + javaParameters.setModulePath(false); + javaParameters.setJvmArgs("-xms:50m"); + javaParameters.setMainArgs("-host 127.0.0.1 -port 8080"); + ResourceInfo resourceJar = new ResourceInfo(); + resourceJar.setId(2); + resourceJar.setResourceName("/opt/share/jar/resource2.jar"); + resourceJar.setRes("I'm resource2.jar"); + ArrayList resourceInfoArrayList = new ArrayList<>(); + resourceInfoArrayList.add(resourceJar); + javaParameters.setResourceList(resourceInfoArrayList); + javaParameters.setRawScript( + "import java.io.IOException;\n" + + "public class JavaTaskTest {\n" + + " public static void main(String[] args) throws IOException {\n" + + " StringBuilder builder = new StringBuilder(\"Hello: \");\n" + + " for (String arg : args) {\n" + + " builder.append(arg).append(\" \");\n" + + " }\n" + " System.out.println(builder);\n" + + " }\n" + + "}\n"); + ArrayList localParams = new ArrayList<>(); + Property property = new Property(); + property.setProp("name"); + property.setValue("zhangsan"); + property.setDirect(IN); + property.setType(VARCHAR); + javaParameters.setLocalParams(localParams); + ResourceInfo mainJar = new ResourceInfo(); + mainJar.setId(1); + mainJar.setResourceName("/opt/share/jar/main.jar"); + mainJar.setRes("I'm main.jar"); + javaParameters.setMainJar(mainJar); + return javaParameters; + } + + /** + * A Java task that constructs the Java runtime pattern + * + * @return JavaTask + **/ + public JavaTask runJavaType() { + TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); + taskExecutionContext.setTaskParams(JSONUtils.toJsonString(createJavaParametersObject(RUN_TYPE_JAVA))); + taskExecutionContext.setExecutePath("/tmp/dolphinscheduler/test/executepath"); + taskExecutionContext.setTaskAppId("runJavaType"); + JavaTask javaTask = new JavaTask(taskExecutionContext); + javaTask.init(); + return javaTask; + } + + /** + * The Java task to construct the jar run mode + * + * @return JavaTask + **/ + public JavaTask runJarType() { + TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); + taskExecutionContext.setTaskParams(JSONUtils.toJsonString(createJavaParametersObject(RUN_TYPE_JAR))); + taskExecutionContext.setExecutePath("/tmp/dolphinscheduler/test/executepath"); + taskExecutionContext.setTaskAppId("runJavaType"); + JavaTask javaTask = new JavaTask(taskExecutionContext); + javaTask.init(); + return javaTask; + } +} diff --git a/dolphinscheduler-task-plugin/pom.xml b/dolphinscheduler-task-plugin/pom.xml index 528e21f459..744a285826 100644 --- a/dolphinscheduler-task-plugin/pom.xml +++ b/dolphinscheduler-task-plugin/pom.xml @@ -56,6 +56,7 @@ dolphinscheduler-task-openmldb dolphinscheduler-task-dvc dolphinscheduler-task-dinky + dolphinscheduler-task-java dolphinscheduler-task-sagemaker dolphinscheduler-task-chunjun dolphinscheduler-task-flink-stream diff --git a/dolphinscheduler-ui/public/images/task-icons/java.png b/dolphinscheduler-ui/public/images/task-icons/java.png new file mode 100644 index 0000000000..218201404e Binary files /dev/null and b/dolphinscheduler-ui/public/images/task-icons/java.png differ diff --git a/dolphinscheduler-ui/public/images/task-icons/java_hover.png b/dolphinscheduler-ui/public/images/task-icons/java_hover.png new file mode 100644 index 0000000000..04eb74b399 Binary files /dev/null and b/dolphinscheduler-ui/public/images/task-icons/java_hover.png differ diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts index fa599a466c..549e6092de 100644 --- a/dolphinscheduler-ui/src/locales/en_US/project.ts +++ b/dolphinscheduler-ui/src/locales/en_US/project.ts @@ -131,7 +131,6 @@ export default { switch_version: 'Switch To This Version', confirm_switch_version: 'Confirm Switch To This Version?', current_version: 'Current Version', - run_type: 'Run Type', scheduling_time: 'Scheduling Time', duration: 'Duration', run_times: 'Run Times', @@ -217,6 +216,7 @@ export default { workflow_state: 'Workflow State', version: 'Version', current_version: 'Current Version', + run_type: 'Run Type', switch_version: 'Switch To This Version', confirm_switch_version: 'Confirm Switch To This Version?', description: 'Description', @@ -309,6 +309,10 @@ export default { online: 'Online' }, node: { + jvm_args: 'Java VM Parameters', + jvm_args_tips: 'Please enter virtual machine parameters', + run_type: 'Run Type', + is_module_path: 'Use Module Path', return_back: 'Return', current_node_settings: 'Current node settings', instructions: 'Instructions', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts index a08d0dc678..ea019f24cc 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts @@ -310,6 +310,10 @@ export default { online: '已上线' }, node: { + is_module_path: '使用模块路径', + run_type: '运行类型', + jvm_args: '虚拟机参数', + jvm_args_tips: '请输入虚拟机参数', return_back: '返回上一节点', current_node_settings: '当前节点设置', instructions: '使用说明', diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/index.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/index.ts index d5235a9fbf..f37b79779d 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/index.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/index.ts @@ -49,6 +49,7 @@ export { useExecutorCores } from './use-executor-cores' export { useMainJar } from './use-main-jar' export { useResources } from './use-resources' export { useTaskDefinition } from './use-task-definition' +export { useJavaTaskMainJar } from './use-java-task-main-jar' export { useShell } from './use-shell' export { useSpark } from './use-spark' @@ -74,6 +75,7 @@ export { useOpenmldb } from './use-openmldb' export { useDvc } from './use-dvc' export { useDinky } from './use-dinky' export { useSagemaker } from './use-sagemaker' +export { useJava } from './use-java' export { useChunjun } from './use-chunjun' export { useChunjunDeployMode } from './use-chunjun-deploy-mode' export { usePytorch } from './use-pytorch' diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-java-task-main-jar.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-java-task-main-jar.ts new file mode 100644 index 0000000000..3fe1fea9ed --- /dev/null +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-java-task-main-jar.ts @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { computed, ref, onMounted, watch } from 'vue' +import { useI18n } from 'vue-i18n' +import { queryResourceByProgramType } from '@/service/modules/resources' +import { useTaskNodeStore } from '@/store/project/task-node' +import utils from '@/utils' +import type { IJsonItem, ProgramType, IMainJar } from '../types' + +export function useJavaTaskMainJar(model: { [field: string]: any }): IJsonItem { + const { t } = useI18n() + const mainJarOptions = ref([] as IMainJar[]) + const taskStore = useTaskNodeStore() + + const mainJarSpan = computed(() => (model.runType === 'JAVA' ? 0 : 24)) + const getMainJars = async (programType: ProgramType) => { + const storeMainJar = taskStore.getMainJar(programType) + if (storeMainJar) { + mainJarOptions.value = storeMainJar + return + } + const res = await queryResourceByProgramType({ + type: 'FILE', + programType + }) + utils.removeUselessChildren(res) + mainJarOptions.value = res || [] + taskStore.updateMainJar(programType, res) + } + + onMounted(() => { + getMainJars(model.programType) + }) + + watch( + () => model.programType, + (value) => { + getMainJars(value) + } + ) + + return { + type: 'tree-select', + field: 'mainJar', + name: t('project.node.main_package'), + span: mainJarSpan, + props: { + cascade: true, + showPath: true, + checkStrategy: 'child', + placeholder: t('project.node.main_package_tips'), + keyField: 'id', + labelField: 'fullName' + }, + validate: { + trigger: ['input', 'blur'], + required: true, + validator(validate: any, value: string) { + if (!value) { + return new Error(t('project.node.main_package_tips')) + } + } + }, + options: mainJarOptions + } +} diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-java.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-java.ts new file mode 100644 index 0000000000..3b05a9c16d --- /dev/null +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-java.ts @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { computed } from 'vue' +import { useI18n } from 'vue-i18n' +import { useCustomParams, useResources ,useJavaTaskMainJar} from '.' +import type { IJsonItem } from '../types' + +export function useJava(model: { [field: string]: any }): IJsonItem[] { + const { t } = useI18n() + const rawScriptSpan = computed(() => (model.runType === 'JAR' ? 0 : 24)) + return [ + { + type: 'select', + field: 'runType', + span: 12, + name: t('project.node.run_type'), + options: RUN_TYPES, + value: model.runType + }, + { + type: 'switch', + field: 'isModulePath', + span: 24, + name: t('project.node.is_module_path'), + value: model.isModulePath + }, + { + type: 'input', + field: 'mainArgs', + name: t('project.node.main_arguments'), + props: { + type: 'textarea', + placeholder: t('project.node.main_arguments_tips') + } + }, + { + type: 'input', + field: 'jvmArgs', + name: t('project.node.jvm_args'), + props: { + type: 'textarea', + placeholder: t('project.node.jvm_args_tips') + } + }, + useJavaTaskMainJar(model), + { + type: 'editor', + field: 'rawScript', + span: rawScriptSpan, + name: t('project.node.script'), + validate: { + trigger: ['input', 'trigger'], + required: true, + message: t('project.node.script_tips') + } + }, + useResources(), + ...useCustomParams({ model, field: 'localParams', isSimple: false }) + ] +} + +export const RUN_TYPES = [ + { + label: 'JAVA', + value: 'JAVA' + }, + { + label: 'JAR', + value: 'JAR' + } + ] + \ No newline at end of file diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts index 0d845fafc1..8c3a1497fc 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts @@ -35,6 +35,17 @@ export function formatParams(data: INodeData): { if (data.taskType === 'SUB_PROCESS') { taskParams.processDefinitionCode = data.processDefinitionCode } + + if(data.taskType === 'JAVA'){ + taskParams.runType = data.runType + taskParams.mainArgs = data.mainArgs + taskParams.jvmArgs = data.jvmArgs + taskParams.isModulePath = data.isModulePath + if(data.runType === 'JAR' && data.mainJar){ + taskParams.mainJar = {id: data.mainJar}; + } + } + if ( data.taskType && ['SPARK', 'MR', 'FLINK', 'FLINK_STREAM'].includes(data.taskType) diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/index.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/index.ts index 1217de2aeb..4ae4d5b14a 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/index.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/index.ts @@ -40,6 +40,7 @@ import { useJupyter } from './use-jupyter' import { useMlflow } from './use-mlflow' import { useOpenmldb } from './use-openmldb' import { useDvc } from './use-dvc' +import { useJava } from './use-java' import { useDinky } from './use-dinky' import { userSagemaker } from './use-sagemaker' import { useChunjun } from './use-chunjun' @@ -75,6 +76,7 @@ export default { SAGEMAKER: userSagemaker, CHUNJUN: useChunjun, FLINK_STREAM: useFlinkStream, + JAVA: useJava, PYTORCH: usePytorch, HIVECLI: useHiveCli } diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-java.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-java.ts new file mode 100644 index 0000000000..822ec193cc --- /dev/null +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-java.ts @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { reactive } from 'vue' +import * as Fields from '../fields/index' +import type { IJsonItem, INodeData } from '../types' +import { ITaskData } from '../types' + +export function useJava({ + projectCode, + from = 0, + readonly, + data +}: { + projectCode: number + from?: number + readonly?: boolean + data?: ITaskData +}) { + const model = reactive({ + name: '', + taskType: 'JAVA', + flag: 'YES', + description: '', + localParams: [], + environmentCode: null, + failRetryInterval: 1, + failRetryTimes: 0, + workerGroup: 'default', + delayTime: 0, + isModulePath: false, + rawScript: '', + timeoutFlag: false, + timeoutNotifyStrategy: ['WARN'], + timeout: 30, + mainJar: undefined, + runType:'JAVA', + mainArgs:'', + jvmArgs:'', + programType: 'JAVA' + } as unknown as INodeData) + + let extra: IJsonItem[] = [] + if (from === 1) { + extra = [ + Fields.useTaskType(model, readonly), + Fields.useProcessName({ + model, + projectCode, + isCreate: !data?.id, + from, + processName: data?.processName + }) + ] + } + + return { + json: [ + Fields.useName(from), + ...extra, + Fields.useRunFlag(), + Fields.useDescription(), + Fields.useTaskPriority(), + Fields.useWorkerGroup(), + Fields.useEnvironmentName(model, !model.id), + ...Fields.useTaskGroup(model, projectCode), + ...Fields.useFailed(), + Fields.useDelayTime(model), + ...Fields.useTimeoutAlarm(model), + ...Fields.useJava(model), + Fields.usePreTasks() + ] as IJsonItem[], + model + } +} diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts index 2dba623ea6..2d3451da08 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts @@ -223,6 +223,9 @@ interface ITaskParams { resourceList?: ISourceItem[] mainJar?: ISourceItem localParams?: ILocalParam[] + runType?:string + jvmArgs?:string + isModulePath?:boolean rawScript?: string initScript?: string programType?: string diff --git a/dolphinscheduler-ui/src/views/projects/task/constants/task-type.ts b/dolphinscheduler-ui/src/views/projects/task/constants/task-type.ts index 0ecc53bccc..d69f38788f 100644 --- a/dolphinscheduler-ui/src/views/projects/task/constants/task-type.ts +++ b/dolphinscheduler-ui/src/views/projects/task/constants/task-type.ts @@ -39,6 +39,7 @@ export type TaskType = | 'MLFLOW' | 'OPENMLDB' | 'DVC' + | 'JAVA' | 'DINKY' | 'SAGEMAKER' | 'CHUNJUN' @@ -49,6 +50,9 @@ export type TaskType = export type TaskExecuteType = 'STREAM' | 'BATCH' export const TASK_TYPES_MAP = { + JAVA: { + alias: 'JAVA' + }, SHELL: { alias: 'SHELL' }, diff --git a/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag.module.scss b/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag.module.scss index d8b4a3da28..8c27b71785 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag.module.scss +++ b/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag.module.scss @@ -177,6 +177,9 @@ $bgLight: #ffffff; &.icon-dvc { background-image: url('/images/task-icons/dvc.png'); } + &.icon-java { + background-image: url('/images/task-icons/java.png'); + } &.icon-dinky { background-image: url('/images/task-icons/dinky.png'); } @@ -272,6 +275,9 @@ $bgLight: #ffffff; &.icon-dinky { background-image: url('/images/task-icons/dinky_hover.png'); } + &.icon-java { + background-image: url('/images/task-icons/java.png'); + } &.icon-sagemaker { background-image: url('/images/task-icons/sagemaker_hover.png'); }