Browse Source

Merge pull request #61 from qiaozhanwei/branch-1.0.0

Python task specifies path execution
pull/2/head
乔占卫 6 years ago committed by GitHub
parent
commit
3d43869642
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 70
      escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java
  2. 65
      escheduler-server/src/test/java/cn/escheduler/server/worker/EnvFileTest.java

70
escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java

@ -19,9 +19,9 @@ package cn.escheduler.server.worker.task;
import cn.escheduler.common.utils.FileUtils; import cn.escheduler.common.utils.FileUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File; import java.io.*;
import java.io.IOException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Paths; import java.nio.file.Paths;
@ -34,6 +34,8 @@ import java.util.function.Consumer;
*/ */
public class PythonCommandExecutor extends AbstractCommandExecutor { public class PythonCommandExecutor extends AbstractCommandExecutor {
private static final Logger logger = LoggerFactory.getLogger(PythonCommandExecutor.class);
public static final String PYTHON = "python"; public static final String PYTHON = "python";
@ -63,27 +65,13 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
*/ */
@Override @Override
protected void createCommandFileIfNotExists(String execCommand, String commandFile) throws IOException { protected void createCommandFileIfNotExists(String execCommand, String commandFile) throws IOException {
logger.info("proxy user:{}, work dir:{}", tenantCode, taskDir); logger.info("tenant :{}, work dir:{}", tenantCode, taskDir);
if (!Files.exists(Paths.get(commandFile))) { if (!Files.exists(Paths.get(commandFile))) {
logger.info("generate command file:{}", commandFile); logger.info("generate command file:{}", commandFile);
StringBuilder sb = new StringBuilder(200); StringBuilder sb = new StringBuilder(200);
sb.append("#-*- encoding=utf8 -*-\n"); sb.append("#-*- encoding=utf8 -*-\n");
sb.append("import os,sys\n");
sb.append("BASEDIR = os.path.dirname(os.path.realpath(__file__))\n");
sb.append("os.chdir(BASEDIR)\n");
if (StringUtils.isNotEmpty(envFile)) {
String[] envArray = envFile.split("\\.");
if(envArray.length == 2){
String path = envArray[0];
logger.info("path:"+path);
int index = path.lastIndexOf("/");
sb.append(String.format("sys.path.append('%s')\n",path.substring(0,index)));
sb.append(String.format("import %s\n",path.substring(index+1)));
}
}
sb.append("\n\n"); sb.append("\n\n");
sb.append(String.format("import py_%s_node\n",taskAppId)); sb.append(String.format("import py_%s_node\n",taskAppId));
@ -96,7 +84,12 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
@Override @Override
protected String commandType() { protected String commandType() {
return PYTHON; String envPath = System.getProperty("user.dir")+"/conf/env/.escheduler_env.sh";
String pythonHome = getPythonHome(envPath);
if (StringUtils.isEmpty(pythonHome)){
return PYTHON;
}
return pythonHome;
} }
@Override @Override
@ -109,4 +102,45 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
return true; return true;
} }
/**
* get python home
* @param envPath
* @return
*/
private static String getPythonHome(String envPath){
BufferedReader br = null;
String line = null;
StringBuilder sb = new StringBuilder();
try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(envPath)));
while ((line = br.readLine()) != null){
if (line.contains("PYTHON_HOME")){
sb.append(line);
break;
}
}
String result = sb.toString();
if (org.apache.commons.lang.StringUtils.isEmpty(result)){
return null;
}
String[] arrs = result.split("=");
if (arrs.length == 2){
return arrs[1];
}
}catch (IOException e){
logger.error("read file failed : " + e.getMessage(),e);
}finally {
try {
if (br != null){
br.close();
}
} catch (IOException e) {
logger.error(e.getMessage(),e);
}
}
return null;
}
} }

65
escheduler-server/src/test/java/cn/escheduler/server/worker/EnvFileTest.java

@ -0,0 +1,65 @@
package cn.escheduler.server.worker;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
/**
* Created by qiaozhanwei on 2019/4/15.
*/
public class EnvFileTest {
private static final Logger logger = LoggerFactory.getLogger(EnvFileTest.class);
public static void main(String[] args) {
String path = System.getProperty("user.dir")+"\\script\\env\\.escheduler_env.sh";
String pythonHome = getPythonHome(path);
logger.info(pythonHome);
}
/**
* get python home
* @param path
* @return
*/
private static String getPythonHome(String path){
BufferedReader br = null;
String line = null;
StringBuilder sb = new StringBuilder();
try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(path)));
while ((line = br.readLine()) != null){
if (line.contains("PYTHON_HOME")){
sb.append(line);
break;
}
}
String result = sb.toString();
if (StringUtils.isEmpty(result)){
return null;
}
String[] arrs = result.split("=");
if (arrs.length == 2){
return arrs[1];
}
}catch (IOException e){
logger.error("read file failed : " + e.getMessage(),e);
}finally {
try {
if (br != null){
br.close();
}
} catch (IOException e) {
logger.error(e.getMessage(),e);
}
}
return null;
}
}
Loading…
Cancel
Save