Merge pull request #63 from qiaozhanwei/branch-1.0.0

Python task specifies path execution update

escheduler-common/src/main/java/cn/escheduler/common/Constants.java

@@ -70,25 +70,6 @@ public final class Constants {
      */
     public static final String YARN_APPLICATION_STATUS_ADDRESS = "yarn.application.status.address";
 
-    /**
-     * spring.redis.maxIdle
-     */
-    public static final String SPRING_REDIS_MAXIDLE = "spring.redis.maxIdle";
-
-    /**
-     * spring.redis.maxTotal
-     */
-    public static final String SPRING_REDIS_MAXTOTAL = "spring.redis.maxTotal";
-
-    /**
-     * spring.redis.host
-     */
-    public static final String SPRING_REDIS_HOST = "spring.redis.host";
-
-    /**
-     * spring.redis.port
-     */
-    public static final String SPRING_REDIS_PORT = "spring.redis.port";
 
     /**
      * hdfs configuration
@@ -117,9 +98,14 @@ public final class Constants {
     public static final String ESCHEDULER_ENV_PATH = "escheduler.env.path";
 
     /**
-     * escheduler.env.py
+     * escheduler.env.sh
      */
-    public static final String ESCHEDULER_ENV_PY = "escheduler.env.py";
+    public static final String ESCHEDULER_ENV_SH = ".escheduler_env.sh";
+
+    /**
+     * python home
+     */
+    public static final String PYTHON_HOME="PYTHON_HOME";
 
     /**
      * resource.view.suffixs
@@ -255,8 +241,6 @@ public final class Constants {
 
     public static final String SCHEDULER_QUEUE_IMPL = "escheduler.queue.impl";
 
-    public static final String SCHEDULER_QUEUE_REDIS_IMPL = "redis";
-
 
     /**
      * date format of yyyy-MM-dd HH:mm:ss

escheduler-common/src/main/java/cn/escheduler/common/utils/CommonUtils.java

@@ -46,13 +46,6 @@ public class CommonUtils {
     return envPath;
   }
 
-  /**
-   * @return get the path of Python system environment variables
-   */
-  public static String getPythonSystemEnvPath() {
-    return getString(ESCHEDULER_ENV_PY);
-  }
-
   /**
    * @return get queue implementation name
    */

escheduler-common/src/main/resources/common/common.properties

@@ -18,7 +18,6 @@ hdfs.startup.state=true
 
 # system env path. self configuration, please make sure the directory and file exists and have read write execute permissions
 escheduler.env.path=/opt/.escheduler_env.sh
-escheduler.env.py=/opt/escheduler_env.py
 
 #resource.view.suffixs
 resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml

escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java

@@ -16,6 +16,7 @@
  */
 package cn.escheduler.server.worker.task;
 
+import cn.escheduler.common.Constants;
 import cn.escheduler.common.utils.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
@@ -84,7 +85,9 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
 
     @Override
     protected String commandType() {
-        String envPath = System.getProperty("user.dir")+"/conf/env/.escheduler_env.sh";
+
+        String envPath = System.getProperty("user.dir") + Constants.SINGLE_SLASH + "conf "+
+                Constants.SINGLE_SLASH +"env" + Constants.SINGLE_SLASH + Constants.ESCHEDULER_ENV_SH;
         String pythonHome = getPythonHome(envPath);
         if (StringUtils.isEmpty(pythonHome)){
             return PYTHON;
@@ -115,7 +118,7 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
         try {
             br = new BufferedReader(new InputStreamReader(new FileInputStream(envPath)));
             while ((line = br.readLine()) != null){
-                if (line.contains("PYTHON_HOME")){
+                if (line.contains(Constants.PYTHON_HOME)){
                     sb.append(line);
                     break;
                 }

escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java

@@ -72,7 +72,7 @@ public class PythonTask extends AbstractTask {
 
     this.pythonProcessTask = new PythonCommandExecutor(this::logHandle,
             taskProps.getTaskDir(), taskProps.getTaskAppId(),
-            taskProps.getTenantCode(), CommonUtils.getPythonSystemEnvPath(), taskProps.getTaskStartTime(),
+            taskProps.getTenantCode(), null, taskProps.getTaskStartTime(),
             taskProps.getTaskTimeout(), logger);
     this.processDao = DaoFactory.getDaoInstance(ProcessDao.class);
   }