Merge remote-tracking branch 'upstream/dev-1.1.0' into dev-1.1.0

5 years ago · 5d19384f61
19 changed files with 107 additions and 44 deletions
--- a/escheduler-alert/pom.xml
+++ b/escheduler-alert/pom.xml
@ -4,7 +4,7 @@
    <parent>
        <groupId>cn.analysys</groupId>
        <artifactId>escheduler</artifactId>
-        <version>1.0.4-SNAPSHOT</version>
+        <version>1.1.0-SNAPSHOT</version>
    </parent>
    <artifactId>escheduler-alert</artifactId>
    <packaging>jar</packaging>
--- a/escheduler-api/pom.xml
+++ b/escheduler-api/pom.xml
@ -1,10 +1,9 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <parent>
    <groupId>cn.analysys</groupId>
    <artifactId>escheduler</artifactId>
-    <version>1.0.4-SNAPSHOT</version>
+    <version>1.1.0-SNAPSHOT</version>
  </parent>
  <artifactId>escheduler-api</artifactId>
  <packaging>jar</packaging>
--- a/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessInstanceService.java
+++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessInstanceService.java
@ -115,12 +115,18 @@ public class ProcessInstanceService extends BaseDAGService {
            return checkResult;
        }
        ProcessInstance processInstance = processDao.findProcessInstanceDetailById(processId);
+        String workerGroupName = "";
        if(processInstance.getWorkerGroupId() == -1){
-            processInstance.setWorkerGroupName(DEFAULT);
+            workerGroupName = DEFAULT;
        }else{
            WorkerGroup workerGroup = workerGroupMapper.queryById(processInstance.getWorkerGroupId());
-            processInstance.setWorkerGroupName(workerGroup.getName());
+            if(workerGroup != null){
+                workerGroupName = DEFAULT;
+            }else{
+                workerGroupName = workerGroup.getName();
+            }
        }
+        processInstance.setWorkerGroupName(workerGroupName);
        ProcessDefinition processDefinition = processDao.findProcessDefineById(processInstance.getProcessDefinitionId());
        processInstance.setReceivers(processDefinition.getReceivers());
        processInstance.setReceiversCc(processDefinition.getReceiversCc());
--- a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java
+++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java
@ -369,7 +369,12 @@ public class ResourcesService extends BaseService {
    public Map<String, Object> queryResourceList(User loginUser, ResourceType type) {

        Map<String, Object> result = new HashMap<>(5);
-        List<Resource> resourceList = resourcesMapper.queryResourceListAuthored(loginUser.getId(), type.ordinal());
+        List<Resource> resourceList;
+        if(isAdmin(loginUser)){
+            resourceList = resourcesMapper.listAllResourceByType(type.ordinal());
+        }else{
+            resourceList = resourcesMapper.queryResourceListAuthored(loginUser.getId(), type.ordinal());
+        }
        result.put(Constants.DATA_LIST, resourceList);
        putMsg(result,Status.SUCCESS);

--- a/escheduler-common/pom.xml
+++ b/escheduler-common/pom.xml
@ -4,7 +4,7 @@
 	<parent>
 		<artifactId>escheduler</artifactId>
 		<groupId>cn.analysys</groupId>
-		<version>1.0.4-SNAPSHOT</version>
+		<version>1.1.0-SNAPSHOT</version>
 	</parent>
 	<artifactId>escheduler-common</artifactId>
 	<name>escheduler-common</name>
--- a/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java
+++ b/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java
@ -22,6 +22,7 @@ import cn.escheduler.common.utils.Bytes;
 import cn.escheduler.common.utils.IpUtils;
 import cn.escheduler.common.utils.OSUtils;
 import cn.escheduler.common.zk.AbstractZKClient;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.data.Stat;
@ -157,7 +158,7 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue {
                    String taskDetail = list.get(i);
                    String[] taskDetailArrs = taskDetail.split(Constants.UNDERLINE);

-                    //向前版本兼容
+                    //forward compatibility 向前版本兼容
                    if(taskDetailArrs.length >= 4){

                        //format ${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskId}
@ -209,17 +210,36 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue {
        while(iterator.hasNext()){
            if(j++ < tasksNum){
                String task = iterator.next();
-                String[] taskArray = task.split(Constants.UNDERLINE);
-                int processInstanceId = Integer.parseInt(taskArray[1]);
-                int taskId = Integer.parseInt(taskArray[3]);
-                String destTask = taskArray[0]+Constants.UNDERLINE + processInstanceId + Constants.UNDERLINE
-                        + taskArray[2] + Constants.UNDERLINE + taskId;
-                taskslist.add(destTask);
+
+                taskslist.add(getOriginTaskFormat(task));
            }
        }
        return taskslist;
    }

+    /**
+     * format ${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskId}
+     * processInstanceId and task id need to be convert to int.
+     * @param formatTask
+     * @return
+     */
+    private String getOriginTaskFormat(String formatTask){
+        String[] taskArray = formatTask.split(Constants.UNDERLINE);
+        int processInstanceId = Integer.parseInt(taskArray[1]);
+        int taskId = Integer.parseInt(taskArray[3]);
+
+        StringBuilder sb = new StringBuilder(50);
+        String destTask = String.format("%s_%s_%s_%s", taskArray[0], processInstanceId, taskArray[3], taskId);
+
+        sb.append(destTask);
+
+        if(taskArray.length > 4){
+            for(int index = 4; index < taskArray.length; index++){
+                sb.append(Constants.UNDERLINE).append(taskArray[index]);
+            }
+        }
+        return sb.toString();
+    }

    @Override
    public void removeNode(String key, String nodeValue){
--- a/escheduler-dao/pom.xml
+++ b/escheduler-dao/pom.xml
@ -4,7 +4,7 @@
 	<parent>
 		<groupId>cn.analysys</groupId>
 		<artifactId>escheduler</artifactId>
-		<version>1.0.4-SNAPSHOT</version>
+		<version>1.1.0-SNAPSHOT</version>
 	</parent>
 	<artifactId>escheduler-dao</artifactId>
 	<name>escheduler-dao</name>
--- a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java
+++ b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java
@ -493,7 +493,8 @@ public class ProcessDao extends AbstractBaseDao {
        processInstance.setProcessInstanceJson(processDefinition.getProcessDefinitionJson());
        // set process instance priority
        processInstance.setProcessInstancePriority(command.getProcessInstancePriority());
-        processInstance.setWorkerGroupId(command.getWorkerGroupId());
+        int workerGroupId = command.getWorkerGroupId() == 0 ? -1 : command.getWorkerGroupId();
+        processInstance.setWorkerGroupId(workerGroupId);
        processInstance.setTimeout(processDefinition.getTimeout());
        processInstance.setTenantId(processDefinition.getTenantId());
        return processInstance;
--- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ResourceMapper.java
+++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ResourceMapper.java
@ -274,5 +274,21 @@ public interface ResourceMapper {
    @SelectProvider(type = ResourceMapperProvider.class, method = "queryTenantCodeByResourceName")
    String queryTenantCodeByResourceName(@Param("resName") String  resName);

-
+  /**
+   * query resource list that the appointed user has permission
+   * @param type
+   * @return
+   */
+  @Results(value = {@Result(property = "id", column = "id", id = true, javaType = int.class, jdbcType = JdbcType.INTEGER),
+          @Result(property = "alias", column = "alias", javaType = String.class, jdbcType = JdbcType.VARCHAR),
+          @Result(property = "fileName", column = "file_name", javaType = String.class, jdbcType = JdbcType.VARCHAR),
+          @Result(property = "desc", column = "desc", javaType = String.class, jdbcType = JdbcType.VARCHAR),
+          @Result(property = "userId", column = "user_id", javaType = int.class, jdbcType = JdbcType.INTEGER),
+          @Result(property = "type", column = "type", typeHandler = EnumOrdinalTypeHandler.class, javaType = ResourceType.class, jdbcType = JdbcType.TINYINT),
+          @Result(property = "size", column = "size", javaType = Long.class, jdbcType = JdbcType.BIGINT),
+          @Result(property = "createTime", column = "create_time", javaType = Timestamp.class, jdbcType = JdbcType.DATE),
+          @Result(property = "updateTime", column = "update_time", javaType = Timestamp.class, jdbcType = JdbcType.DATE)
+  })
+  @SelectProvider(type = ResourceMapperProvider.class, method = "listAllResourceByType")
+  List<Resource> listAllResourceByType(@Param("type") Integer type);
 }
--- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ResourceMapperProvider.java
+++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ResourceMapperProvider.java
@ -295,4 +295,19 @@ public class ResourceMapperProvider {
      WHERE("type = #{type} and user_id = #{userId}");
    }}.toString();
  }
+
+  /**
+   * list all resource by type
+   *
+   * @param parameter
+   * @return
+   */
+  public String listAllResourceByType(Map<String, Object> parameter) {
+    return new SQL() {{
+      SELECT("*");
+      FROM(TABLE_NAME);
+      WHERE("type = #{type}");
+    }}.toString();
+  }
+
 }
--- a/escheduler-rpc/pom.xml
+++ b/escheduler-rpc/pom.xml
@ -4,7 +4,7 @@
    <parent>
        <artifactId>escheduler</artifactId>
        <groupId>cn.analysys</groupId>
-        <version>1.0.4-SNAPSHOT</version>
+        <version>1.1.0-SNAPSHOT</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

--- a/escheduler-server/pom.xml
+++ b/escheduler-server/pom.xml
@ -3,7 +3,7 @@
 	<parent>
 		<artifactId>escheduler</artifactId>
 		<groupId>cn.analysys</groupId>
-		<version>1.0.4-SNAPSHOT</version>
+		<version>1.1.0-SNAPSHOT</version>
 	</parent>
 	<artifactId>escheduler-server</artifactId>
 	<name>escheduler-server</name>
--- a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java
+++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java
@ -153,7 +153,7 @@ public class FetchTaskThread implements Runnable{
                                }

                                String[] taskStringArray = taskQueueStr.split(Constants.UNDERLINE);
-                                String taskInstIdStr = taskStringArray[taskStringArray.length - 1];
+                                String taskInstIdStr = taskStringArray[3];
                                Date now = new Date();
                                Integer taskId = Integer.parseInt(taskInstIdStr);

--- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java
+++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java
@ -18,6 +18,7 @@ package cn.escheduler.server.worker.task;

 import cn.escheduler.common.Constants;
 import cn.escheduler.common.utils.FileUtils;
+import cn.escheduler.common.utils.PropertyUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@ -71,11 +72,11 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
        if (!Files.exists(Paths.get(commandFile))) {
            logger.info("generate command file:{}", commandFile);

-            StringBuilder sb = new StringBuilder(200);
+            StringBuilder sb = new StringBuilder();
            sb.append("#-*- encoding=utf8 -*-\n");

            sb.append("\n\n");
-            sb.append(String.format("import py_%s_node\n",taskAppId));
+            sb.append(execCommand);
            logger.info(sb.toString());

            // write data to file
@ -86,8 +87,8 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
    @Override
    protected String commandType() {

-        String envPath = System.getProperty("user.dir") + Constants.SINGLE_SLASH + "conf"+
-                Constants.SINGLE_SLASH +"env" + Constants.SINGLE_SLASH + Constants.ESCHEDULER_ENV_SH;
+        String envPath = PropertyUtils.getString(Constants.ESCHEDULER_ENV_PATH);
+
        String pythonHome = getPythonHome(envPath);
        if (StringUtils.isEmpty(pythonHome)){
            return PYTHON;
--- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java
+++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java
@ -112,14 +112,14 @@ public class PythonTask extends AbstractTask {
   */
  private String buildCommand() throws Exception {
    // generate scripts
-    String fileName = String.format("%s/py_%s_node.py", taskDir, taskProps.getTaskAppId());
-    Path path = new File(fileName).toPath();
+//    String fileName = String.format("%s/py_%s_node.py", taskDir, taskProps.getTaskAppId());
+//    Path path = new File(fileName).toPath();



-    if (Files.exists(path)) {
-      return fileName;
-    }
+//    if (Files.exists(path)) {
+//      return fileName;
+//    }

    String rawScript = pythonParameters.getRawScript().replaceAll("\\r\\n", "\n");

@ -140,19 +140,20 @@ public class PythonTask extends AbstractTask {
    }


-    pythonParameters.setRawScript(rawScript);
+//    pythonParameters.setRawScript(rawScript);

    logger.info("raw script : {}", pythonParameters.getRawScript());
    logger.info("task dir : {}", taskDir);

-    Set<PosixFilePermission> perms = PosixFilePermissions.fromString("rwxr-xr-x");
-    FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms);
-
-    Files.createFile(path, attr);
-
-    Files.write(path, pythonParameters.getRawScript().getBytes(), StandardOpenOption.APPEND);
-
-    return fileName;
+//    Set<PosixFilePermission> perms = PosixFilePermissions.fromString("rwxr-xr-x");
+//    FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms);
+//
+//    Files.createFile(path, attr);
+//
+//    Files.write(path, pythonParameters.getRawScript().getBytes(), StandardOpenOption.APPEND);
+//
+//    return fileName;
+    return rawScript;
  }

  @Override
--- a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue
+++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue
@ -21,7 +21,7 @@
      </div>
    </div>
    <div class="clearfix list">
-      <x-button type="info"  style="margin-left:20px" shape="circle" :loading="spinnerLoading" @click="preview()" v-ps="['GENERAL_USER']">执行时间</x-button>
+      <x-button type="info"  style="margin-left:20px" shape="circle" :loading="spinnerLoading" @click="preview()">执行时间</x-button>
      <div class="text">
        {{$t('Timing')}}
      </div>
@ -136,7 +136,7 @@
    </div>
    <div class="submit">
      <x-button type="text" @click="close()"> {{$t('Cancel')}} </x-button>
-      <x-button type="primary" shape="circle" :loading="spinnerLoading" @click="ok()" v-ps="['GENERAL_USER']">{{spinnerLoading ? 'Loading...' : (item.crontab ? $t('Edit') : $t('Create'))}} </x-button>
+      <x-button type="primary" shape="circle" :loading="spinnerLoading" @click="ok()">{{spinnerLoading ? 'Loading...' : (item.crontab ? $t('Edit') : $t('Create'))}} </x-button>
    </div>
  </div>
 </template>
--- a/pom.xml
+++ b/pom.xml
@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>cn.analysys</groupId>
 	<artifactId>escheduler</artifactId>
-	<version>1.0.4-SNAPSHOT</version>
+	<version>1.1.0-SNAPSHOT</version>
 	<packaging>pom</packaging>
 	<name>escheduler</name>
 	<url>http://maven.apache.org</url>
--- a/sql/soft_version
+++ b/sql/soft_version
@ -1 +1 @@
-1.0.4
+1.1.0
--- a/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql
+++ b/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql
@ -1 +0,0 @@
-INSERT INTO `t_escheduler_version` (`version`) VALUES ('1.1.0');
 @ -1 +1 @@
 .0.4
 .1.0
				`@ -1 +0,0 @@`
				INSERT INTO `t_escheduler_version` (`version`) VALUES ('1.1.0');