Browse Source

Fix jar path is not correct in java task (#15906)

3.2.2-release-bak
Wenjun Ruan 7 months ago committed by GitHub
parent
commit
0a11cd21bd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 1
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/resource/ResourceContext.java
  2. 4
      dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java
  3. 41
      dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java
  4. 32
      dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java
  5. 1
      dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java

1
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/resource/ResourceContext.java

@ -60,7 +60,6 @@ public class ResourceContext {
public static class ResourceItem { public static class ResourceItem {
private String resourceAbsolutePathInStorage; private String resourceAbsolutePathInStorage;
private String resourceRelativePath;
private String resourceAbsolutePathInLocal; private String resourceAbsolutePathInLocal;
} }

4
dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java

@ -65,7 +65,7 @@ public class HiveCliTaskTest {
} }
@Test @Test
public void hiveCliTaskExecuteSqlFromScript() throws Exception { public void hiveCliTaskExecuteSqlFromScript() {
String hiveCliTaskParameters = buildHiveCliTaskExecuteSqlFromScriptParameters(); String hiveCliTaskParameters = buildHiveCliTaskExecuteSqlFromScriptParameters();
HiveCliTask hiveCliTask = prepareHiveCliTaskForTest(hiveCliTaskParameters); HiveCliTask hiveCliTask = prepareHiveCliTaskForTest(hiveCliTaskParameters);
hiveCliTask.init(); hiveCliTask.init();
@ -78,7 +78,7 @@ public class HiveCliTaskTest {
TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); TaskExecutionContext taskExecutionContext = new TaskExecutionContext();
taskExecutionContext.setTaskParams(hiveCliTaskParameters); taskExecutionContext.setTaskParams(hiveCliTaskParameters);
ResourceContext resourceContext = new ResourceContext(); ResourceContext resourceContext = new ResourceContext();
resourceContext.addResourceItem(new ResourceContext.ResourceItem("/sql_tasks/hive_task.sql", "123_node.sql", resourceContext.addResourceItem(new ResourceContext.ResourceItem("/sql_tasks/hive_task.sql",
"/sql_tasks/hive_task.sql")); "/sql_tasks/hive_task.sql"));
taskExecutionContext.setResourceContext(resourceContext); taskExecutionContext.setResourceContext(resourceContext);

41
dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java

@ -88,6 +88,7 @@ public class JavaTask extends AbstractTask {
/** /**
* Initializes a Java task * Initializes a Java task
*
* @return void * @return void
**/ **/
@Override @Override
@ -178,7 +179,8 @@ public class JavaTask extends AbstractTask {
**/ **/
protected String buildJarCommand() { protected String buildJarCommand() {
ResourceContext resourceContext = taskRequest.getResourceContext(); ResourceContext resourceContext = taskRequest.getResourceContext();
String mainJarName = resourceContext.getResourceItem(javaParameters.getMainJar().getResourceName()) String mainJarAbsolutePathInLocal = resourceContext
.getResourceItem(javaParameters.getMainJar().getResourceName())
.getResourceAbsolutePathInLocal(); .getResourceAbsolutePathInLocal();
StringBuilder builder = new StringBuilder(); StringBuilder builder = new StringBuilder();
builder.append(getJavaCommandPath()) builder.append(getJavaCommandPath())
@ -186,7 +188,7 @@ public class JavaTask extends AbstractTask {
.append(buildResourcePath()).append(" ") .append(buildResourcePath()).append(" ")
.append("-jar").append(" ") .append("-jar").append(" ")
.append(taskRequest.getExecutePath()).append(FOLDER_SEPARATOR) .append(taskRequest.getExecutePath()).append(FOLDER_SEPARATOR)
.append(mainJarName).append(" ") .append(mainJarAbsolutePathInLocal).append(" ")
.append(javaParameters.getMainArgs().trim()).append(" ") .append(javaParameters.getMainArgs().trim()).append(" ")
.append(javaParameters.getJvmArgs().trim()); .append(javaParameters.getJvmArgs().trim());
return builder.toString(); return builder.toString();
@ -207,39 +209,6 @@ public class JavaTask extends AbstractTask {
return javaParameters; return javaParameters;
} }
/**
* Replaces placeholders such as local variables in source files
*
* @param rawScript
* @return String
* @throws StringIndexOutOfBoundsException
*/
protected static String convertJavaSourceCodePlaceholders(String rawScript) throws StringIndexOutOfBoundsException {
int len = "${setShareVar(${".length();
int scriptStart = 0;
while ((scriptStart = rawScript.indexOf("${setShareVar(${", scriptStart)) != -1) {
int start = -1;
int end = rawScript.indexOf('}', scriptStart + len);
String prop = rawScript.substring(scriptStart + len, end);
start = rawScript.indexOf(',', end);
end = rawScript.indexOf(')', start);
String value = rawScript.substring(start + 1, end);
start = rawScript.indexOf('}', start) + 1;
end = rawScript.length();
String replaceScript = String.format("print(\"${{setValue({},{})}}\".format(\"%s\",%s))", prop, value);
rawScript = rawScript.substring(0, scriptStart) + replaceScript + rawScript.substring(start, end);
scriptStart += replaceScript.length();
}
return rawScript;
}
/** /**
* Creates a Java source file when it does not exist * Creates a Java source file when it does not exist
* *
@ -290,8 +259,6 @@ public class JavaTask extends AbstractTask {
for (ResourceInfo info : javaParameters.getResourceFilesList()) { for (ResourceInfo info : javaParameters.getResourceFilesList()) {
builder.append(JavaConstants.PATH_SEPARATOR); builder.append(JavaConstants.PATH_SEPARATOR);
builder builder
.append(taskRequest.getExecutePath())
.append(FOLDER_SEPARATOR)
.append(resourceContext.getResourceItem(info.getResourceName()).getResourceAbsolutePathInLocal()); .append(resourceContext.getResourceItem(info.getResourceName()).getResourceAbsolutePathInLocal());
} }
return builder.toString(); return builder.toString();

32
dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java

@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.plugin.task.java; package org.apache.dolphinscheduler.plugin.task.java;
import static com.google.common.truth.Truth.assertThat;
import static org.apache.dolphinscheduler.plugin.task.api.enums.DataType.VARCHAR; import static org.apache.dolphinscheduler.plugin.task.api.enums.DataType.VARCHAR;
import static org.apache.dolphinscheduler.plugin.task.api.enums.Direct.IN; import static org.apache.dolphinscheduler.plugin.task.api.enums.Direct.IN;
import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_JAR; import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_JAR;
@ -34,7 +35,6 @@ import org.apache.dolphinscheduler.plugin.task.java.exception.JavaSourceFileExis
import org.apache.dolphinscheduler.plugin.task.java.exception.PublicClassNotFoundException; import org.apache.dolphinscheduler.plugin.task.java.exception.PublicClassNotFoundException;
import org.apache.dolphinscheduler.plugin.task.java.exception.RunTypeNotFoundException; import org.apache.dolphinscheduler.plugin.task.java.exception.RunTypeNotFoundException;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.lang.reflect.Field; import java.lang.reflect.Field;
import java.nio.file.Files; import java.nio.file.Files;
@ -82,10 +82,10 @@ public class JavaTaskTest {
**/ **/
@Test @Test
public void buildJarCommand() { public void buildJarCommand() {
String homeBinPath = JavaConstants.JAVA_HOME_VAR + File.separator + "bin" + File.separator;
JavaTask javaTask = runJarType(); JavaTask javaTask = runJarType();
Assertions.assertEquals(javaTask.buildJarCommand(), homeBinPath assertThat(javaTask.buildJarCommand())
+ "java -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar -jar /tmp/dolphinscheduler/test/executepath/opt/share/jar/main.jar -host 127.0.0.1 -port 8080 -xms:50m"); .isEqualTo(
"${JAVA_HOME}/bin/java -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar -jar /tmp/dolphinscheduler/test/executepath/opt/share/jar/main.jar -host 127.0.0.1 -port 8080 -xms:50m");
} }
/** /**
@ -101,14 +101,13 @@ public class JavaTaskTest {
Assertions.assertEquals("JavaTaskTest", publicClassName); Assertions.assertEquals("JavaTaskTest", publicClassName);
String fileName = javaTask.buildJavaSourceCodeFileFullName(publicClassName); String fileName = javaTask.buildJavaSourceCodeFileFullName(publicClassName);
try { try {
String homeBinPath = JavaConstants.JAVA_HOME_VAR + File.separator + "bin" + File.separator;
Path path = Paths.get(fileName); Path path = Paths.get(fileName);
if (Files.exists(path)) { if (Files.exists(path)) {
Files.delete(path); Files.delete(path);
} }
Assertions.assertEquals(homeBinPath assertThat(javaTask.buildJavaCompileCommand(sourceCode))
+ "javac -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java", .isEqualTo(
javaTask.buildJavaCompileCommand(sourceCode)); "${JAVA_HOME}/bin/javac -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java");
} finally { } finally {
Path path = Paths.get(fileName); Path path = Paths.get(fileName);
if (Files.exists(path)) { if (Files.exists(path)) {
@ -121,26 +120,29 @@ public class JavaTaskTest {
/** /**
* Construct java to run the command * Construct java to run the command
* *
* @return void * @return void
**/ **/
@Test @Test
public void buildJavaCommand() throws Exception { public void buildJavaCommand() throws Exception {
String wantJavaCommand =
"${JAVA_HOME}/bin/javac -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java;${JAVA_HOME}/bin/java -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar JavaTaskTest -host 127.0.0.1 -port 8080 -xms:50m";
JavaTask javaTask = runJavaType(); JavaTask javaTask = runJavaType();
String sourceCode = javaTask.buildJavaSourceContent(); String sourceCode = javaTask.buildJavaSourceContent();
String publicClassName = javaTask.getPublicClassName(sourceCode); String publicClassName = javaTask.getPublicClassName(sourceCode);
Assertions.assertEquals("JavaTaskTest", publicClassName); Assertions.assertEquals("JavaTaskTest", publicClassName);
String fileName = javaTask.buildJavaSourceCodeFileFullName(publicClassName); String fileName = javaTask.buildJavaSourceCodeFileFullName(publicClassName);
Path path = Paths.get(fileName); Path path = Paths.get(fileName);
if (Files.exists(path)) { if (Files.exists(path)) {
Files.delete(path); Files.delete(path);
} }
Assertions.assertEquals(wantJavaCommand, javaTask.buildJavaCommand()); assertThat(javaTask.buildJavaCommand())
.isEqualTo(
"${JAVA_HOME}/bin/javac -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java;${JAVA_HOME}/bin/java -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar JavaTaskTest -host 127.0.0.1 -port 8080 -xms:50m");
} }
/** /**
* There is no exception to overwriting the Java source file * There is no exception to overwriting the Java source file
*
* @return void * @return void
* @throws IOException * @throws IOException
**/ **/
@ -259,8 +261,8 @@ public class JavaTaskTest {
resourceItem2.setResourceAbsolutePathInLocal("opt/share/jar/main.jar"); resourceItem2.setResourceAbsolutePathInLocal("opt/share/jar/main.jar");
ResourceContext.ResourceItem resourceItem3 = new ResourceContext.ResourceItem(); ResourceContext.ResourceItem resourceItem3 = new ResourceContext.ResourceItem();
resourceItem2.setResourceAbsolutePathInStorage("/JavaTaskTest.java"); resourceItem3.setResourceAbsolutePathInStorage("/JavaTaskTest.java");
resourceItem2.setResourceAbsolutePathInLocal("JavaTaskTest.java"); resourceItem3.setResourceAbsolutePathInLocal("JavaTaskTest.java");
ResourceContext resourceContext = new ResourceContext(); ResourceContext resourceContext = new ResourceContext();
resourceContext.addResourceItem(resourceItem1); resourceContext.addResourceItem(resourceItem1);
@ -275,7 +277,7 @@ public class JavaTaskTest {
/** /**
* The Java task to construct the jar run mode * The Java task to construct the jar run mode
* *
* @return JavaTask * @return JavaTask
**/ **/
private JavaTask runJarType() { private JavaTask runJarType() {
TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); TaskExecutionContext taskExecutionContext = new TaskExecutionContext();

1
dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java

@ -151,7 +151,6 @@ public class TaskExecutionContextUtils {
} }
ResourceContext.ResourceItem resourceItem = ResourceContext.ResourceItem.builder() ResourceContext.ResourceItem resourceItem = ResourceContext.ResourceItem.builder()
.resourceAbsolutePathInStorage(resourceAbsolutePathInStorage) .resourceAbsolutePathInStorage(resourceAbsolutePathInStorage)
.resourceRelativePath(resourceRelativePath)
.resourceAbsolutePathInLocal(resourceAbsolutePathInLocal) .resourceAbsolutePathInLocal(resourceAbsolutePathInLocal)
.build(); .build();
resourceContext.addResourceItem(resourceItem); resourceContext.addResourceItem(resourceItem);

Loading…
Cancel
Save