Browse Source

[Fix-5596][Python] Fix conflict between python_home and datax_home configuration in dolphinscheduler_env.sh (#5612)

2.0.7-release
Shiwen Cheng 4 years ago committed by GitHub
parent
commit
8bf042ae6e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java
  2. 24
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java
  3. 16
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java

@ -141,7 +141,7 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
if (PYTHON_PATH_PATTERN.matcher(pythonHome).find()) { if (PYTHON_PATH_PATTERN.matcher(pythonHome).find()) {
return pythonHome; return pythonHome;
} }
return pythonHome + "/bin/python"; return Paths.get(pythonHome, "/bin/python").toString();
} }
/** /**

24
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java

@ -31,6 +31,7 @@ import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.server.entity.DataxTaskExecutionContext; import org.apache.dolphinscheduler.server.entity.DataxTaskExecutionContext;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import org.apache.dolphinscheduler.server.utils.DataxUtils; import org.apache.dolphinscheduler.server.utils.DataxUtils;
@ -45,6 +46,7 @@ import java.io.File;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption; import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermission;
@ -58,6 +60,8 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -86,6 +90,7 @@ public class DataxTask extends AbstractTask {
* python process(datax only supports version 2.7 by default) * python process(datax only supports version 2.7 by default)
*/ */
private static final String DATAX_PYTHON = "python2.7"; private static final String DATAX_PYTHON = "python2.7";
private static final Pattern PYTHON_PATH_PATTERN = Pattern.compile("/bin/python[\\d.]*$");
/** /**
* datax path * datax path
*/ */
@ -392,7 +397,7 @@ public class DataxTask extends AbstractTask {
// datax python command // datax python command
StringBuilder sbr = new StringBuilder(); StringBuilder sbr = new StringBuilder();
sbr.append(DATAX_PYTHON); sbr.append(getPythonCommand());
sbr.append(" "); sbr.append(" ");
sbr.append(DATAX_PATH); sbr.append(DATAX_PATH);
sbr.append(" "); sbr.append(" ");
@ -419,6 +424,23 @@ public class DataxTask extends AbstractTask {
return fileName; return fileName;
} }
public String getPythonCommand() {
String pythonHome = System.getenv("PYTHON_HOME");
return getPythonCommand(pythonHome);
}
public String getPythonCommand(String pythonHome) {
if (StringUtils.isEmpty(pythonHome)) {
return DATAX_PYTHON;
}
String pythonBinPath = "/bin/" + DATAX_PYTHON;
Matcher matcher = PYTHON_PATH_PATTERN.matcher(pythonHome);
if (matcher.find()) {
return matcher.replaceAll(pythonBinPath);
}
return Paths.get(pythonHome, pythonBinPath).toString();
}
public String loadJvmEnv(DataxParameters dataXParameters) { public String loadJvmEnv(DataxParameters dataXParameters) {
int xms = dataXParameters.getXms() < 1 ? 1 : dataXParameters.getXms(); int xms = dataXParameters.getXms() < 1 ? 1 : dataXParameters.getXms();
int xmx = dataXParameters.getXmx() < 1 ? 1 : dataXParameters.getXmx(); int xmx = dataXParameters.getXmx() < 1 ? 1 : dataXParameters.getXmx();

16
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java

@ -455,6 +455,22 @@ public class DataxTaskTest {
} }
} }
@Test
public void testGetPythonCommand() {
String pythonCommand = dataxTask.getPythonCommand();
Assert.assertEquals("python2.7", pythonCommand);
pythonCommand = dataxTask.getPythonCommand("");
Assert.assertEquals("python2.7", pythonCommand);
pythonCommand = dataxTask.getPythonCommand("/usr/bin/python");
Assert.assertEquals("/usr/bin/python2.7", pythonCommand);
pythonCommand = dataxTask.getPythonCommand("/usr/local/bin/python2");
Assert.assertEquals("/usr/local/bin/python2.7", pythonCommand);
pythonCommand = dataxTask.getPythonCommand("/opt/python/bin/python3.8");
Assert.assertEquals("/opt/python/bin/python2.7", pythonCommand);
pythonCommand = dataxTask.getPythonCommand("/opt/soft/python");
Assert.assertEquals("/opt/soft/python/bin/python2.7", pythonCommand);
}
@Test @Test
public void testLoadJvmEnv() { public void testLoadJvmEnv() {
DataxTask dataxTask = new DataxTask(null,null); DataxTask dataxTask = new DataxTask(null,null);

Loading…
Cancel
Save