From 0a11cd21bd0e73b52d4e68dae2f32519031e2e50 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Mon, 29 Apr 2024 18:07:05 +0800 Subject: [PATCH] Fix jar path is not correct in java task (#15906) --- .../task/api/resource/ResourceContext.java | 1 - .../plugin/task/hivecli/HiveCliTaskTest.java | 4 +- .../plugin/task/java/JavaTask.java | 41 ++----------------- .../plugin/task/java/JavaTaskTest.java | 32 ++++++++------- .../utils/TaskExecutionContextUtils.java | 1 - 5 files changed, 23 insertions(+), 56 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/resource/ResourceContext.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/resource/ResourceContext.java index 687d1aeb95..f90b526902 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/resource/ResourceContext.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/resource/ResourceContext.java @@ -60,7 +60,6 @@ public class ResourceContext { public static class ResourceItem { private String resourceAbsolutePathInStorage; - private String resourceRelativePath; private String resourceAbsolutePathInLocal; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java index 824ad49f89..b4136af3c3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java @@ -65,7 +65,7 @@ public class HiveCliTaskTest { } @Test - public void hiveCliTaskExecuteSqlFromScript() throws Exception { + public void hiveCliTaskExecuteSqlFromScript() { String hiveCliTaskParameters = buildHiveCliTaskExecuteSqlFromScriptParameters(); HiveCliTask hiveCliTask = prepareHiveCliTaskForTest(hiveCliTaskParameters); hiveCliTask.init(); @@ -78,7 +78,7 @@ public class HiveCliTaskTest { TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); taskExecutionContext.setTaskParams(hiveCliTaskParameters); ResourceContext resourceContext = new ResourceContext(); - resourceContext.addResourceItem(new ResourceContext.ResourceItem("/sql_tasks/hive_task.sql", "123_node.sql", + resourceContext.addResourceItem(new ResourceContext.ResourceItem("/sql_tasks/hive_task.sql", "/sql_tasks/hive_task.sql")); taskExecutionContext.setResourceContext(resourceContext); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java index 179b50c35c..fc23260345 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java @@ -88,6 +88,7 @@ public class JavaTask extends AbstractTask { /** * Initializes a Java task + * * @return void **/ @Override @@ -178,7 +179,8 @@ public class JavaTask extends AbstractTask { **/ protected String buildJarCommand() { ResourceContext resourceContext = taskRequest.getResourceContext(); - String mainJarName = resourceContext.getResourceItem(javaParameters.getMainJar().getResourceName()) + String mainJarAbsolutePathInLocal = resourceContext + .getResourceItem(javaParameters.getMainJar().getResourceName()) .getResourceAbsolutePathInLocal(); StringBuilder builder = new StringBuilder(); builder.append(getJavaCommandPath()) @@ -186,7 +188,7 @@ public class JavaTask extends AbstractTask { .append(buildResourcePath()).append(" ") .append("-jar").append(" ") .append(taskRequest.getExecutePath()).append(FOLDER_SEPARATOR) - .append(mainJarName).append(" ") + .append(mainJarAbsolutePathInLocal).append(" ") .append(javaParameters.getMainArgs().trim()).append(" ") .append(javaParameters.getJvmArgs().trim()); return builder.toString(); @@ -207,39 +209,6 @@ public class JavaTask extends AbstractTask { return javaParameters; } - /** - * Replaces placeholders such as local variables in source files - * - * @param rawScript - * @return String - * @throws StringIndexOutOfBoundsException - */ - protected static String convertJavaSourceCodePlaceholders(String rawScript) throws StringIndexOutOfBoundsException { - int len = "${setShareVar(${".length(); - - int scriptStart = 0; - while ((scriptStart = rawScript.indexOf("${setShareVar(${", scriptStart)) != -1) { - int start = -1; - int end = rawScript.indexOf('}', scriptStart + len); - String prop = rawScript.substring(scriptStart + len, end); - - start = rawScript.indexOf(',', end); - end = rawScript.indexOf(')', start); - - String value = rawScript.substring(start + 1, end); - - start = rawScript.indexOf('}', start) + 1; - end = rawScript.length(); - - String replaceScript = String.format("print(\"${{setValue({},{})}}\".format(\"%s\",%s))", prop, value); - - rawScript = rawScript.substring(0, scriptStart) + replaceScript + rawScript.substring(start, end); - - scriptStart += replaceScript.length(); - } - return rawScript; - } - /** * Creates a Java source file when it does not exist * @@ -290,8 +259,6 @@ public class JavaTask extends AbstractTask { for (ResourceInfo info : javaParameters.getResourceFilesList()) { builder.append(JavaConstants.PATH_SEPARATOR); builder - .append(taskRequest.getExecutePath()) - .append(FOLDER_SEPARATOR) .append(resourceContext.getResourceItem(info.getResourceName()).getResourceAbsolutePathInLocal()); } return builder.toString(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java index 55756241ce..c829415326 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.plugin.task.java; +import static com.google.common.truth.Truth.assertThat; import static org.apache.dolphinscheduler.plugin.task.api.enums.DataType.VARCHAR; import static org.apache.dolphinscheduler.plugin.task.api.enums.Direct.IN; import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_JAR; @@ -34,7 +35,6 @@ import org.apache.dolphinscheduler.plugin.task.java.exception.JavaSourceFileExis import org.apache.dolphinscheduler.plugin.task.java.exception.PublicClassNotFoundException; import org.apache.dolphinscheduler.plugin.task.java.exception.RunTypeNotFoundException; -import java.io.File; import java.io.IOException; import java.lang.reflect.Field; import java.nio.file.Files; @@ -82,10 +82,10 @@ public class JavaTaskTest { **/ @Test public void buildJarCommand() { - String homeBinPath = JavaConstants.JAVA_HOME_VAR + File.separator + "bin" + File.separator; JavaTask javaTask = runJarType(); - Assertions.assertEquals(javaTask.buildJarCommand(), homeBinPath - + "java -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar -jar /tmp/dolphinscheduler/test/executepath/opt/share/jar/main.jar -host 127.0.0.1 -port 8080 -xms:50m"); + assertThat(javaTask.buildJarCommand()) + .isEqualTo( + "${JAVA_HOME}/bin/java -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar -jar /tmp/dolphinscheduler/test/executepath/opt/share/jar/main.jar -host 127.0.0.1 -port 8080 -xms:50m"); } /** @@ -101,14 +101,13 @@ public class JavaTaskTest { Assertions.assertEquals("JavaTaskTest", publicClassName); String fileName = javaTask.buildJavaSourceCodeFileFullName(publicClassName); try { - String homeBinPath = JavaConstants.JAVA_HOME_VAR + File.separator + "bin" + File.separator; Path path = Paths.get(fileName); if (Files.exists(path)) { Files.delete(path); } - Assertions.assertEquals(homeBinPath - + "javac -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java", - javaTask.buildJavaCompileCommand(sourceCode)); + assertThat(javaTask.buildJavaCompileCommand(sourceCode)) + .isEqualTo( + "${JAVA_HOME}/bin/javac -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java"); } finally { Path path = Paths.get(fileName); if (Files.exists(path)) { @@ -121,26 +120,29 @@ public class JavaTaskTest { /** * Construct java to run the command * - * @return void + * @return void **/ @Test public void buildJavaCommand() throws Exception { - String wantJavaCommand = - "${JAVA_HOME}/bin/javac -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java;${JAVA_HOME}/bin/java -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar JavaTaskTest -host 127.0.0.1 -port 8080 -xms:50m"; JavaTask javaTask = runJavaType(); String sourceCode = javaTask.buildJavaSourceContent(); String publicClassName = javaTask.getPublicClassName(sourceCode); + Assertions.assertEquals("JavaTaskTest", publicClassName); + String fileName = javaTask.buildJavaSourceCodeFileFullName(publicClassName); Path path = Paths.get(fileName); if (Files.exists(path)) { Files.delete(path); } - Assertions.assertEquals(wantJavaCommand, javaTask.buildJavaCommand()); + assertThat(javaTask.buildJavaCommand()) + .isEqualTo( + "${JAVA_HOME}/bin/javac -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java;${JAVA_HOME}/bin/java -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar JavaTaskTest -host 127.0.0.1 -port 8080 -xms:50m"); } /** * There is no exception to overwriting the Java source file + * * @return void * @throws IOException **/ @@ -259,8 +261,8 @@ public class JavaTaskTest { resourceItem2.setResourceAbsolutePathInLocal("opt/share/jar/main.jar"); ResourceContext.ResourceItem resourceItem3 = new ResourceContext.ResourceItem(); - resourceItem2.setResourceAbsolutePathInStorage("/JavaTaskTest.java"); - resourceItem2.setResourceAbsolutePathInLocal("JavaTaskTest.java"); + resourceItem3.setResourceAbsolutePathInStorage("/JavaTaskTest.java"); + resourceItem3.setResourceAbsolutePathInLocal("JavaTaskTest.java"); ResourceContext resourceContext = new ResourceContext(); resourceContext.addResourceItem(resourceItem1); @@ -275,7 +277,7 @@ public class JavaTaskTest { /** * The Java task to construct the jar run mode * - * @return JavaTask + * @return JavaTask **/ private JavaTask runJarType() { TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java index f43b19c444..8d83dde593 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java @@ -151,7 +151,6 @@ public class TaskExecutionContextUtils { } ResourceContext.ResourceItem resourceItem = ResourceContext.ResourceItem.builder() .resourceAbsolutePathInStorage(resourceAbsolutePathInStorage) - .resourceRelativePath(resourceRelativePath) .resourceAbsolutePathInLocal(resourceAbsolutePathInLocal) .build(); resourceContext.addResourceItem(resourceItem);