Merge remote-tracking branch 'upstream/dev-1.1.0' into dev-1.1.0

escheduler-alert/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>cn.analysys</groupId>
         <artifactId>escheduler</artifactId>
-        <version>1.0.4-SNAPSHOT</version>
+        <version>1.1.0-SNAPSHOT</version>
     </parent>
     <artifactId>escheduler-alert</artifactId>
     <packaging>jar</packaging>

escheduler-api/pom.xml

@@ -1,10 +1,9 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>cn.analysys</groupId>
     <artifactId>escheduler</artifactId>
-    <version>1.0.4-SNAPSHOT</version>
+    <version>1.1.0-SNAPSHOT</version>
   </parent>
   <artifactId>escheduler-api</artifactId>
   <packaging>jar</packaging>
@@ -48,6 +47,10 @@
           <groupId>org.springframework.boot</groupId>
           <artifactId>spring-boot-starter-tomcat</artifactId>
         </exclusion>
+        <exclusion>
+          <artifactId>log4j-to-slf4j</artifactId>
+          <groupId>org.apache.logging.log4j</groupId>
+        </exclusion>
       </exclusions>
     </dependency>
 

escheduler-common/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<artifactId>escheduler</artifactId>
 		<groupId>cn.analysys</groupId>
-		<version>1.0.4-SNAPSHOT</version>
+		<version>1.1.0-SNAPSHOT</version>
 	</parent>
 	<artifactId>escheduler-common</artifactId>
 	<name>escheduler-common</name>

escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java

@@ -151,7 +151,27 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue {
                 int size = list.size();
 
 
-                Set<String> taskTreeSet = new TreeSet<>();
+                Set<String> taskTreeSet = new TreeSet<>(new Comparator<String>() {
+                    @Override
+                    public int compare(String o1, String o2) {
+
+                        String s1 = o1;
+                        String s2 = o2;
+                        String[] s1Array = s1.split(Constants.UNDERLINE);
+                        if(s1Array.length>4){
+                            // warning: if this length > 5, need to be changed
+                            s1 = s1.substring(0, s1.lastIndexOf(Constants.UNDERLINE) );
+                        }
+
+                        String[] s2Array = s2.split(Constants.UNDERLINE);
+                        if(s2Array.length>4){
+                            // warning: if this length > 5, need to be changed
+                            s2 = s2.substring(0, s2.lastIndexOf(Constants.UNDERLINE) );
+                        }
+
+                        return s1.compareTo(s2);
+                    }
+                });
 
                 for (int i = 0; i < size; i++) {
 
@@ -173,8 +193,8 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue {
                                     continue;
                                 }
                             }
+                            formatTask += Constants.UNDERLINE + taskDetailArrs[4];
                         }
-
                         taskTreeSet.add(formatTask);
 
                     }
@@ -229,7 +249,7 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue {
         int taskId = Integer.parseInt(taskArray[3]);
 
         StringBuilder sb = new StringBuilder(50);
-        String destTask = String.format("%s_%s_%s_%s", taskArray[0], processInstanceId, taskArray[3], taskId);
+        String destTask = String.format("%s_%s_%s_%s", taskArray[0], processInstanceId, taskArray[2], taskId);
 
         sb.append(destTask);
 

escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java

@@ -17,6 +17,8 @@
 package cn.escheduler.common.queue;
 
 import cn.escheduler.common.Constants;
+import cn.escheduler.common.utils.IpUtils;
+import cn.escheduler.common.utils.OSUtils;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -58,31 +60,31 @@ public class TaskQueueImplTest {
     @Test
     public void testAdd(){
 
+
         //add
-        tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_1_1_1_2130706433,3232236775");
+        tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_0_1_1_-1");
         tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"0_1_1_1_2130706433,3232236775");
-        tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_1_0_1_2130706433,3232236775");
+        tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_1_0_1_2130706433,3232236775,"+IpUtils.ipToLong(OSUtils.getHost()));
         tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_2_1_1_2130706433,3232236775");
 
         List<String> tasks = tasksQueue.poll(Constants.SCHEDULER_TASKS_QUEUE, 1);
 
-        if(tasks.size() < 0){
+        if(tasks.size() <= 0){
             return;
         }
 
         //pop
         String node1 = tasks.get(0);
 
-        assertEquals(node1,"0_0000000001_1_0000000001");
+        assertEquals(node1,"1_0_1_1_-1");
 
         tasks = tasksQueue.poll(Constants.SCHEDULER_TASKS_QUEUE, 1);
 
-        if(tasks.size() < 0){
+        if(tasks.size() <= 0){
             return;
         }
 
         String node2 = tasks.get(0);
-        assertEquals(node2,"0_0000000001_1_0000000001");
 
     }
 

escheduler-dao/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>cn.analysys</groupId>
 		<artifactId>escheduler</artifactId>
-		<version>1.0.4-SNAPSHOT</version>
+		<version>1.1.0-SNAPSHOT</version>
 	</parent>
 	<artifactId>escheduler-dao</artifactId>
 	<name>escheduler-dao</name>
@@ -37,6 +37,10 @@
 					<groupId>org.apache.tomcat</groupId>
 					<artifactId>tomcat-jdbc</artifactId>
 				</exclusion>
+				<exclusion>
+					<artifactId>log4j-to-slf4j</artifactId>
+					<groupId>org.apache.logging.log4j</groupId>
+				</exclusion>
 			</exclusions>
 		</dependency>
 		<dependency>

escheduler-rpc/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <artifactId>escheduler</artifactId>
         <groupId>cn.analysys</groupId>
-        <version>1.0.4-SNAPSHOT</version>
+        <version>1.1.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

escheduler-server/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<artifactId>escheduler</artifactId>
 		<groupId>cn.analysys</groupId>
-		<version>1.0.4-SNAPSHOT</version>
+		<version>1.1.0-SNAPSHOT</version>
 	</parent>
 	<artifactId>escheduler-server</artifactId>
 	<name>escheduler-server</name>

escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java

@@ -18,6 +18,7 @@ package cn.escheduler.server.worker.task;
 
 import cn.escheduler.common.Constants;
 import cn.escheduler.common.utils.FileUtils;
+import cn.escheduler.common.utils.PropertyUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -71,11 +72,11 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
         if (!Files.exists(Paths.get(commandFile))) {
             logger.info("generate command file:{}", commandFile);
 
-            StringBuilder sb = new StringBuilder(200);
+            StringBuilder sb = new StringBuilder();
             sb.append("#-*- encoding=utf8 -*-\n");
 
             sb.append("\n\n");
-            sb.append(String.format("import py_%s_node\n",taskAppId));
+            sb.append(execCommand);
             logger.info(sb.toString());
 
             // write data to file
@@ -86,8 +87,8 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
     @Override
     protected String commandType() {
 
-        String envPath = System.getProperty("user.dir") + Constants.SINGLE_SLASH + "conf"+
+        String envPath = PropertyUtils.getString(Constants.ESCHEDULER_ENV_PATH);
-                Constants.SINGLE_SLASH +"env" + Constants.SINGLE_SLASH + Constants.ESCHEDULER_ENV_SH;
+
         String pythonHome = getPythonHome(envPath);
         if (StringUtils.isEmpty(pythonHome)){
             return PYTHON;

escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java

@@ -112,14 +112,14 @@ public class PythonTask extends AbstractTask {
    */
   private String buildCommand() throws Exception {
     // generate scripts
-    String fileName = String.format("%s/py_%s_node.py", taskDir, taskProps.getTaskAppId());
+//    String fileName = String.format("%s/py_%s_node.py", taskDir, taskProps.getTaskAppId());
-    Path path = new File(fileName).toPath();
+//    Path path = new File(fileName).toPath();
 
 
 
-    if (Files.exists(path)) {
+//    if (Files.exists(path)) {
-      return fileName;
+//      return fileName;
-    }
+//    }
 
     String rawScript = pythonParameters.getRawScript().replaceAll("\\r\\n", "\n");
 
@@ -140,19 +140,20 @@ public class PythonTask extends AbstractTask {
     }
 
 
-    pythonParameters.setRawScript(rawScript);
+//    pythonParameters.setRawScript(rawScript);
 
     logger.info("raw script : {}", pythonParameters.getRawScript());
     logger.info("task dir : {}", taskDir);
 
-    Set<PosixFilePermission> perms = PosixFilePermissions.fromString("rwxr-xr-x");
+//    Set<PosixFilePermission> perms = PosixFilePermissions.fromString("rwxr-xr-x");
-    FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms);
+//    FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms);
-
+//
-    Files.createFile(path, attr);
+//    Files.createFile(path, attr);
-
+//
-    Files.write(path, pythonParameters.getRawScript().getBytes(), StandardOpenOption.APPEND);
+//    Files.write(path, pythonParameters.getRawScript().getBytes(), StandardOpenOption.APPEND);
-
+//
-    return fileName;
+//    return fileName;
+    return rawScript;
   }
 
   @Override

escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue

@@ -16,7 +16,7 @@
         <div class="col-md-3">
           <div class="text-num-model text">
             <div class="title">
-              <span >{{$t('failure command number')}}}</span>
+              <span >{{$t('failure command number')}}</span>
             </div>
             <div class="value-p">
               <b :style="{color:color[1]}"> {{commandCountData.errorCount}}</b>

escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue

@@ -254,11 +254,6 @@
 
                 this.store.dispatch(api, searchParams).then(res => {
                   this.previewTimes = res
-                  if (this.previewTimes.length) {
-                    resolve()
-                  } else {
-                    reject(new Error(0))
-                  }
                 })
               }
             },

escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue

@@ -131,7 +131,8 @@
         }
       },
       _verification () {
-        let regEmail = /^([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+@([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+\.[a-zA-Z]{2,3}$/ // eslint-disable-line
+        let regEmail = /^([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+@([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+\.[a-zA-Z]{2,}$/ // eslint-disable-line
+
         // Mobile phone number regular
         let regPhone = /^1(3|4|5|6|7|8)\d{9}$/; // eslint-disable-line
 
@@ -184,7 +185,10 @@
       _getTenantList () {
         return new Promise((resolve, reject) => {
           this.store.dispatch('security/getTenantList').then(res => {
-            this.tenantList = _.map(res, v => {
+            let arr = _.filter(res, (o) => {
+              return o.id !== -1
+            })
+            this.tenantList = _.map(arr, v => {
               return {
                 id: v.id,
                 code: v.tenantName

install.sh

@@ -144,6 +144,7 @@ singleYarnIp="ark1"
 hdfsPath="/escheduler"
 
 # 拥有在hdfs根路径/下创建目录权限的用户
+# 注意：如果开启了kerberos，则直接hdfsRootUser=，就可以
 hdfsRootUser="hdfs"
 
 # common 配置

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>cn.analysys</groupId>
 	<artifactId>escheduler</artifactId>
-	<version>1.0.4-SNAPSHOT</version>
+	<version>1.1.0-SNAPSHOT</version>
 	<packaging>pom</packaging>
 	<name>escheduler</name>
 	<url>http://maven.apache.org</url>

sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql

@@ -1 +0,0 @@
-INSERT INTO `t_escheduler_version` (`version`) VALUES ('1.1.0');