From a7e6f30d554f490c356029c1c25da7793d50c15e Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Tue, 16 Apr 2019 17:36:35 +0800 Subject: [PATCH 1/2] Python task specifies path execution update --- .../src/main/java/cn/escheduler/common/Constants.java | 7 ++++++- .../main/java/cn/escheduler/common/utils/CommonUtils.java | 7 ------- .../src/main/resources/common/common.properties | 1 - .../server/worker/task/PythonCommandExecutor.java | 7 +++++-- .../escheduler/server/worker/task/python/PythonTask.java | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java index e0e0c399e9..b25d0f8798 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java @@ -119,7 +119,12 @@ public final class Constants { /** * escheduler.env.py */ - public static final String ESCHEDULER_ENV_PY = "escheduler.env.py"; + public static final String ESCHEDULER_ENV_SH = ".escheduler_env.sh"; + + /** + * python home + */ + public static final String PYTHON_HOME="PYTHON_HOME"; /** * resource.view.suffixs diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/CommonUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/CommonUtils.java index 6930d8db35..d0164791d2 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/CommonUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/CommonUtils.java @@ -46,13 +46,6 @@ public class CommonUtils { return envPath; } - /** - * @return get the path of Python system environment variables - */ - public static String getPythonSystemEnvPath() { - return getString(ESCHEDULER_ENV_PY); - } - /** * @return get queue implementation name */ diff --git a/escheduler-common/src/main/resources/common/common.properties b/escheduler-common/src/main/resources/common/common.properties index c501ad0d1a..1cb995ba0e 100644 --- a/escheduler-common/src/main/resources/common/common.properties +++ b/escheduler-common/src/main/resources/common/common.properties @@ -18,7 +18,6 @@ hdfs.startup.state=true # system env path. self configuration, please make sure the directory and file exists and have read write execute permissions escheduler.env.path=/opt/.escheduler_env.sh -escheduler.env.py=/opt/escheduler_env.py #resource.view.suffixs resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java index 4ca978cdc1..0374df37d4 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java @@ -16,6 +16,7 @@ */ package cn.escheduler.server.worker.task; +import cn.escheduler.common.Constants; import cn.escheduler.common.utils.FileUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -84,7 +85,9 @@ public class PythonCommandExecutor extends AbstractCommandExecutor { @Override protected String commandType() { - String envPath = System.getProperty("user.dir")+"/conf/env/.escheduler_env.sh"; + + String envPath = System.getProperty("user.dir") + Constants.SINGLE_SLASH + "conf "+ + Constants.SINGLE_SLASH +"env" + Constants.SINGLE_SLASH + Constants.ESCHEDULER_ENV_SH; String pythonHome = getPythonHome(envPath); if (StringUtils.isEmpty(pythonHome)){ return PYTHON; @@ -115,7 +118,7 @@ public class PythonCommandExecutor extends AbstractCommandExecutor { try { br = new BufferedReader(new InputStreamReader(new FileInputStream(envPath))); while ((line = br.readLine()) != null){ - if (line.contains("PYTHON_HOME")){ + if (line.contains(Constants.PYTHON_HOME)){ sb.append(line); break; } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java index 2b7ae29b96..49d754404a 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java @@ -72,7 +72,7 @@ public class PythonTask extends AbstractTask { this.pythonProcessTask = new PythonCommandExecutor(this::logHandle, taskProps.getTaskDir(), taskProps.getTaskAppId(), - taskProps.getTenantCode(), CommonUtils.getPythonSystemEnvPath(), taskProps.getTaskStartTime(), + taskProps.getTenantCode(), null, taskProps.getTaskStartTime(), taskProps.getTaskTimeout(), logger); this.processDao = DaoFactory.getDaoInstance(ProcessDao.class); } From cd8b51adbf03b50ab3513eb9d80e1d33f5b992ef Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Tue, 16 Apr 2019 17:42:41 +0800 Subject: [PATCH 2/2] update Constants --- .../java/cn/escheduler/common/Constants.java | 23 +------------------ 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java index b25d0f8798..40f97c4487 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java @@ -70,25 +70,6 @@ public final class Constants { */ public static final String YARN_APPLICATION_STATUS_ADDRESS = "yarn.application.status.address"; - /** - * spring.redis.maxIdle - */ - public static final String SPRING_REDIS_MAXIDLE = "spring.redis.maxIdle"; - - /** - * spring.redis.maxTotal - */ - public static final String SPRING_REDIS_MAXTOTAL = "spring.redis.maxTotal"; - - /** - * spring.redis.host - */ - public static final String SPRING_REDIS_HOST = "spring.redis.host"; - - /** - * spring.redis.port - */ - public static final String SPRING_REDIS_PORT = "spring.redis.port"; /** * hdfs configuration @@ -117,7 +98,7 @@ public final class Constants { public static final String ESCHEDULER_ENV_PATH = "escheduler.env.path"; /** - * escheduler.env.py + * escheduler.env.sh */ public static final String ESCHEDULER_ENV_SH = ".escheduler_env.sh"; @@ -260,8 +241,6 @@ public final class Constants { public static final String SCHEDULER_QUEUE_IMPL = "escheduler.queue.impl"; - public static final String SCHEDULER_QUEUE_REDIS_IMPL = "redis"; - /** * date format of yyyy-MM-dd HH:mm:ss