Browse Source

fix python home conflict (#12112)

3.2.0-release
JieguangZhou 2 years ago committed by GitHub
parent
commit
6eb1eb722a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PythonEnvManager.java
  2. 12
      dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/test/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskTest.java

4
dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PythonEnvManager.java

@ -40,7 +40,7 @@ public class PythonEnvManager {
private static final String VIRTUALENV_BUILD = "virtualenv -p ${PYTHON_HOME} %s"; private static final String VIRTUALENV_BUILD = "virtualenv -p ${PYTHON_HOME} %s";
private static final String INSTALL_COMMAND = "python -m pip install -r %s"; private static final String INSTALL_COMMAND = "%s -m pip install -r %s";
private String pythonEnvTool = ENV_TOOL_VENV; private String pythonEnvTool = ENV_TOOL_VENV;
@ -54,7 +54,7 @@ public class PythonEnvManager {
} else if (pythonEnvTool.equals(ENV_TOOL_CONDA)) { } else if (pythonEnvTool.equals(ENV_TOOL_CONDA)) {
buildCommand = String.format(CONDA_BUILD, condaPythonVersion, CREATE_ENV_NAME); buildCommand = String.format(CONDA_BUILD, condaPythonVersion, CREATE_ENV_NAME);
} }
String installCommand = String.format(INSTALL_COMMAND, requirementPath); String installCommand = String.format(INSTALL_COMMAND, getPythonCommand(), requirementPath);
return buildCommand + " && " + sourceCommand + " && " + installCommand; return buildCommand + " && " + sourceCommand + " && " + installCommand;
} }

12
dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/test/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskTest.java

@ -72,16 +72,16 @@ public class PytorchTaskTest {
envManager.setPythonEnvTool(PythonEnvManager.ENV_TOOL_CONDA); envManager.setPythonEnvTool(PythonEnvManager.ENV_TOOL_CONDA);
envManager.setCondaPythonVersion("3.9"); envManager.setCondaPythonVersion("3.9");
String condaEnvCommand39 = envManager.getBuildEnvCommand(requirementPath); String condaEnvCommand39 = envManager.getBuildEnvCommand(requirementPath);
Assert.assertEquals(condaEnvCommand39, "conda create -y python=3.9 -p ./venv && source activate ./venv && python -m pip install -r " + requirementPath); Assert.assertEquals(condaEnvCommand39, "conda create -y python=3.9 -p ./venv && source activate ./venv && ./venv/bin/python -m pip install -r " + requirementPath);
envManager.setCondaPythonVersion("3.8"); envManager.setCondaPythonVersion("3.8");
String condaEnvCommand38 = envManager.getBuildEnvCommand(requirementPath); String condaEnvCommand38 = envManager.getBuildEnvCommand(requirementPath);
Assert.assertEquals(condaEnvCommand38, "conda create -y python=3.8 -p ./venv && source activate ./venv && python -m pip install -r " + requirementPath); Assert.assertEquals(condaEnvCommand38, "conda create -y python=3.8 -p ./venv && source activate ./venv && ./venv/bin/python -m pip install -r " + requirementPath);
envManager.setPythonEnvTool(PythonEnvManager.ENV_TOOL_VENV); envManager.setPythonEnvTool(PythonEnvManager.ENV_TOOL_VENV);
String venvEnvCommand = envManager.getBuildEnvCommand(requirementPath); String venvEnvCommand = envManager.getBuildEnvCommand(requirementPath);
Assert.assertEquals(venvEnvCommand, "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && python -m pip install -r " + requirementPath); Assert.assertEquals(venvEnvCommand, "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && ./venv/bin/python -m pip install -r " + requirementPath);
} }
@ -146,7 +146,7 @@ public class PytorchTaskTest {
PytorchTask task = initTask(parameters); PytorchTask task = initTask(parameters);
Assert.assertEquals(task.buildPythonExecuteCommand(), Assert.assertEquals(task.buildPythonExecuteCommand(),
"export PYTHONPATH=.\n" + "export PYTHONPATH=.\n" +
"conda create -y python=3.6 -p ./venv && source activate ./venv && python -m pip install -r requirements.txt\n" + "conda create -y python=3.6 -p ./venv && source activate ./venv && ./venv/bin/python -m pip install -r requirements.txt\n" +
"./venv/bin/python main.py --epochs=1 --dry-run"); "./venv/bin/python main.py --epochs=1 --dry-run");
} }
@ -163,7 +163,7 @@ public class PytorchTaskTest {
PytorchTask task = initTask(parameters); PytorchTask task = initTask(parameters);
Assert.assertEquals(task.buildPythonExecuteCommand(), Assert.assertEquals(task.buildPythonExecuteCommand(),
"export PYTHONPATH=.\n" + "export PYTHONPATH=.\n" +
"virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && python -m pip install -r requirements.txt\n" + "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && ./venv/bin/python -m pip install -r requirements.txt\n" +
"./venv/bin/python main.py --epochs=1 --dry-run"); "./venv/bin/python main.py --epochs=1 --dry-run");
} }
@ -189,7 +189,7 @@ public class PytorchTaskTest {
createFile(scriptFile); createFile(scriptFile);
String expected = "export PYTHONPATH=%s\n" + String expected = "export PYTHONPATH=%s\n" +
"virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && python -m pip install -r %s\n" + "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && ./venv/bin/python -m pip install -r %s\n" +
"./venv/bin/python %s"; "./venv/bin/python %s";
System.out.println(task.buildPythonExecuteCommand()); System.out.println(task.buildPythonExecuteCommand());
Assert.assertEquals(String.format(expected, pythonPath, requirementFile, scriptFile), task.buildPythonExecuteCommand()); Assert.assertEquals(String.format(expected, pythonPath, requirementFile, scriptFile), task.buildPythonExecuteCommand());

Loading…
Cancel
Save