Browse Source

Merge remote-tracking branch 'upstream/dev-1.1.0' into dev-1.1.0

pull/2/head
lidongdai 5 years ago
parent
commit
9f975e0ae8
  1. 2
      escheduler-alert/pom.xml
  2. 9
      escheduler-api/pom.xml
  3. 2
      escheduler-common/pom.xml
  4. 26
      escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java
  5. 14
      escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java
  6. 6
      escheduler-dao/pom.xml
  7. 2
      escheduler-rpc/pom.xml
  8. 2
      escheduler-server/pom.xml
  9. 9
      escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java
  10. 29
      escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java
  11. 2
      escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue
  12. 5
      escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue
  13. 8
      escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue
  14. 1
      install.sh
  15. 2
      pom.xml
  16. 1
      sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql

2
escheduler-alert/pom.xml

@ -4,7 +4,7 @@
<parent> <parent>
<groupId>cn.analysys</groupId> <groupId>cn.analysys</groupId>
<artifactId>escheduler</artifactId> <artifactId>escheduler</artifactId>
<version>1.0.4-SNAPSHOT</version> <version>1.1.0-SNAPSHOT</version>
</parent> </parent>
<artifactId>escheduler-alert</artifactId> <artifactId>escheduler-alert</artifactId>
<packaging>jar</packaging> <packaging>jar</packaging>

9
escheduler-api/pom.xml

@ -1,10 +1,9 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<parent> <parent>
<groupId>cn.analysys</groupId> <groupId>cn.analysys</groupId>
<artifactId>escheduler</artifactId> <artifactId>escheduler</artifactId>
<version>1.0.4-SNAPSHOT</version> <version>1.1.0-SNAPSHOT</version>
</parent> </parent>
<artifactId>escheduler-api</artifactId> <artifactId>escheduler-api</artifactId>
<packaging>jar</packaging> <packaging>jar</packaging>
@ -48,6 +47,10 @@
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId> <artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion> </exclusion>
<exclusion>
<artifactId>log4j-to-slf4j</artifactId>
<groupId>org.apache.logging.log4j</groupId>
</exclusion>
</exclusions> </exclusions>
</dependency> </dependency>

2
escheduler-common/pom.xml

@ -4,7 +4,7 @@
<parent> <parent>
<artifactId>escheduler</artifactId> <artifactId>escheduler</artifactId>
<groupId>cn.analysys</groupId> <groupId>cn.analysys</groupId>
<version>1.0.4-SNAPSHOT</version> <version>1.1.0-SNAPSHOT</version>
</parent> </parent>
<artifactId>escheduler-common</artifactId> <artifactId>escheduler-common</artifactId>
<name>escheduler-common</name> <name>escheduler-common</name>

26
escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java

@ -151,7 +151,27 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue {
int size = list.size(); int size = list.size();
Set<String> taskTreeSet = new TreeSet<>(); Set<String> taskTreeSet = new TreeSet<>(new Comparator<String>() {
@Override
public int compare(String o1, String o2) {
String s1 = o1;
String s2 = o2;
String[] s1Array = s1.split(Constants.UNDERLINE);
if(s1Array.length>4){
// warning: if this length > 5, need to be changed
s1 = s1.substring(0, s1.lastIndexOf(Constants.UNDERLINE) );
}
String[] s2Array = s2.split(Constants.UNDERLINE);
if(s2Array.length>4){
// warning: if this length > 5, need to be changed
s2 = s2.substring(0, s2.lastIndexOf(Constants.UNDERLINE) );
}
return s1.compareTo(s2);
}
});
for (int i = 0; i < size; i++) { for (int i = 0; i < size; i++) {
@ -173,8 +193,8 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue {
continue; continue;
} }
} }
formatTask += Constants.UNDERLINE + taskDetailArrs[4];
} }
taskTreeSet.add(formatTask); taskTreeSet.add(formatTask);
} }
@ -229,7 +249,7 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue {
int taskId = Integer.parseInt(taskArray[3]); int taskId = Integer.parseInt(taskArray[3]);
StringBuilder sb = new StringBuilder(50); StringBuilder sb = new StringBuilder(50);
String destTask = String.format("%s_%s_%s_%s", taskArray[0], processInstanceId, taskArray[3], taskId); String destTask = String.format("%s_%s_%s_%s", taskArray[0], processInstanceId, taskArray[2], taskId);
sb.append(destTask); sb.append(destTask);

14
escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java

@ -17,6 +17,8 @@
package cn.escheduler.common.queue; package cn.escheduler.common.queue;
import cn.escheduler.common.Constants; import cn.escheduler.common.Constants;
import cn.escheduler.common.utils.IpUtils;
import cn.escheduler.common.utils.OSUtils;
import org.junit.After; import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
@ -58,31 +60,31 @@ public class TaskQueueImplTest {
@Test @Test
public void testAdd(){ public void testAdd(){
//add //add
tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_1_1_1_2130706433,3232236775"); tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_0_1_1_-1");
tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"0_1_1_1_2130706433,3232236775"); tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"0_1_1_1_2130706433,3232236775");
tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_1_0_1_2130706433,3232236775"); tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_1_0_1_2130706433,3232236775,"+IpUtils.ipToLong(OSUtils.getHost()));
tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_2_1_1_2130706433,3232236775"); tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_2_1_1_2130706433,3232236775");
List<String> tasks = tasksQueue.poll(Constants.SCHEDULER_TASKS_QUEUE, 1); List<String> tasks = tasksQueue.poll(Constants.SCHEDULER_TASKS_QUEUE, 1);
if(tasks.size() < 0){ if(tasks.size() <= 0){
return; return;
} }
//pop //pop
String node1 = tasks.get(0); String node1 = tasks.get(0);
assertEquals(node1,"0_0000000001_1_0000000001"); assertEquals(node1,"1_0_1_1_-1");
tasks = tasksQueue.poll(Constants.SCHEDULER_TASKS_QUEUE, 1); tasks = tasksQueue.poll(Constants.SCHEDULER_TASKS_QUEUE, 1);
if(tasks.size() < 0){ if(tasks.size() <= 0){
return; return;
} }
String node2 = tasks.get(0); String node2 = tasks.get(0);
assertEquals(node2,"0_0000000001_1_0000000001");
} }

6
escheduler-dao/pom.xml

@ -4,7 +4,7 @@
<parent> <parent>
<groupId>cn.analysys</groupId> <groupId>cn.analysys</groupId>
<artifactId>escheduler</artifactId> <artifactId>escheduler</artifactId>
<version>1.0.4-SNAPSHOT</version> <version>1.1.0-SNAPSHOT</version>
</parent> </parent>
<artifactId>escheduler-dao</artifactId> <artifactId>escheduler-dao</artifactId>
<name>escheduler-dao</name> <name>escheduler-dao</name>
@ -37,6 +37,10 @@
<groupId>org.apache.tomcat</groupId> <groupId>org.apache.tomcat</groupId>
<artifactId>tomcat-jdbc</artifactId> <artifactId>tomcat-jdbc</artifactId>
</exclusion> </exclusion>
<exclusion>
<artifactId>log4j-to-slf4j</artifactId>
<groupId>org.apache.logging.log4j</groupId>
</exclusion>
</exclusions> </exclusions>
</dependency> </dependency>
<dependency> <dependency>

2
escheduler-rpc/pom.xml

@ -4,7 +4,7 @@
<parent> <parent>
<artifactId>escheduler</artifactId> <artifactId>escheduler</artifactId>
<groupId>cn.analysys</groupId> <groupId>cn.analysys</groupId>
<version>1.0.4-SNAPSHOT</version> <version>1.1.0-SNAPSHOT</version>
</parent> </parent>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>

2
escheduler-server/pom.xml

@ -3,7 +3,7 @@
<parent> <parent>
<artifactId>escheduler</artifactId> <artifactId>escheduler</artifactId>
<groupId>cn.analysys</groupId> <groupId>cn.analysys</groupId>
<version>1.0.4-SNAPSHOT</version> <version>1.1.0-SNAPSHOT</version>
</parent> </parent>
<artifactId>escheduler-server</artifactId> <artifactId>escheduler-server</artifactId>
<name>escheduler-server</name> <name>escheduler-server</name>

9
escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java

@ -18,6 +18,7 @@ package cn.escheduler.server.worker.task;
import cn.escheduler.common.Constants; import cn.escheduler.common.Constants;
import cn.escheduler.common.utils.FileUtils; import cn.escheduler.common.utils.FileUtils;
import cn.escheduler.common.utils.PropertyUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -71,11 +72,11 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
if (!Files.exists(Paths.get(commandFile))) { if (!Files.exists(Paths.get(commandFile))) {
logger.info("generate command file:{}", commandFile); logger.info("generate command file:{}", commandFile);
StringBuilder sb = new StringBuilder(200); StringBuilder sb = new StringBuilder();
sb.append("#-*- encoding=utf8 -*-\n"); sb.append("#-*- encoding=utf8 -*-\n");
sb.append("\n\n"); sb.append("\n\n");
sb.append(String.format("import py_%s_node\n",taskAppId)); sb.append(execCommand);
logger.info(sb.toString()); logger.info(sb.toString());
// write data to file // write data to file
@ -86,8 +87,8 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
@Override @Override
protected String commandType() { protected String commandType() {
String envPath = System.getProperty("user.dir") + Constants.SINGLE_SLASH + "conf"+ String envPath = PropertyUtils.getString(Constants.ESCHEDULER_ENV_PATH);
Constants.SINGLE_SLASH +"env" + Constants.SINGLE_SLASH + Constants.ESCHEDULER_ENV_SH;
String pythonHome = getPythonHome(envPath); String pythonHome = getPythonHome(envPath);
if (StringUtils.isEmpty(pythonHome)){ if (StringUtils.isEmpty(pythonHome)){
return PYTHON; return PYTHON;

29
escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java

@ -112,14 +112,14 @@ public class PythonTask extends AbstractTask {
*/ */
private String buildCommand() throws Exception { private String buildCommand() throws Exception {
// generate scripts // generate scripts
String fileName = String.format("%s/py_%s_node.py", taskDir, taskProps.getTaskAppId()); // String fileName = String.format("%s/py_%s_node.py", taskDir, taskProps.getTaskAppId());
Path path = new File(fileName).toPath(); // Path path = new File(fileName).toPath();
if (Files.exists(path)) { // if (Files.exists(path)) {
return fileName; // return fileName;
} // }
String rawScript = pythonParameters.getRawScript().replaceAll("\\r\\n", "\n"); String rawScript = pythonParameters.getRawScript().replaceAll("\\r\\n", "\n");
@ -140,19 +140,20 @@ public class PythonTask extends AbstractTask {
} }
pythonParameters.setRawScript(rawScript); // pythonParameters.setRawScript(rawScript);
logger.info("raw script : {}", pythonParameters.getRawScript()); logger.info("raw script : {}", pythonParameters.getRawScript());
logger.info("task dir : {}", taskDir); logger.info("task dir : {}", taskDir);
Set<PosixFilePermission> perms = PosixFilePermissions.fromString("rwxr-xr-x"); // Set<PosixFilePermission> perms = PosixFilePermissions.fromString("rwxr-xr-x");
FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms); // FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms);
//
Files.createFile(path, attr); // Files.createFile(path, attr);
//
Files.write(path, pythonParameters.getRawScript().getBytes(), StandardOpenOption.APPEND); // Files.write(path, pythonParameters.getRawScript().getBytes(), StandardOpenOption.APPEND);
//
return fileName; // return fileName;
return rawScript;
} }
@Override @Override

2
escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue

@ -16,7 +16,7 @@
<div class="col-md-3"> <div class="col-md-3">
<div class="text-num-model text"> <div class="text-num-model text">
<div class="title"> <div class="title">
<span >{{$t('failure command number')}}}</span> <span >{{$t('failure command number')}}</span>
</div> </div>
<div class="value-p"> <div class="value-p">
<b :style="{color:color[1]}"> {{commandCountData.errorCount}}</b> <b :style="{color:color[1]}"> {{commandCountData.errorCount}}</b>

5
escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue

@ -254,11 +254,6 @@
this.store.dispatch(api, searchParams).then(res => { this.store.dispatch(api, searchParams).then(res => {
this.previewTimes = res this.previewTimes = res
if (this.previewTimes.length) {
resolve()
} else {
reject(new Error(0))
}
}) })
} }
}, },

8
escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue

@ -131,7 +131,8 @@
} }
}, },
_verification () { _verification () {
let regEmail = /^([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+@([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+\.[a-zA-Z]{2,3}$/ // eslint-disable-line let regEmail = /^([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+@([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+\.[a-zA-Z]{2,}$/ // eslint-disable-line
// Mobile phone number regular // Mobile phone number regular
let regPhone = /^1(3|4|5|6|7|8)\d{9}$/; // eslint-disable-line let regPhone = /^1(3|4|5|6|7|8)\d{9}$/; // eslint-disable-line
@ -184,7 +185,10 @@
_getTenantList () { _getTenantList () {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
this.store.dispatch('security/getTenantList').then(res => { this.store.dispatch('security/getTenantList').then(res => {
this.tenantList = _.map(res, v => { let arr = _.filter(res, (o) => {
return o.id !== -1
})
this.tenantList = _.map(arr, v => {
return { return {
id: v.id, id: v.id,
code: v.tenantName code: v.tenantName

1
install.sh

@ -144,6 +144,7 @@ singleYarnIp="ark1"
hdfsPath="/escheduler" hdfsPath="/escheduler"
# 拥有在hdfs根路径/下创建目录权限的用户 # 拥有在hdfs根路径/下创建目录权限的用户
# 注意:如果开启了kerberos,则直接hdfsRootUser=,就可以
hdfsRootUser="hdfs" hdfsRootUser="hdfs"
# common 配置 # common 配置

2
pom.xml

@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>cn.analysys</groupId> <groupId>cn.analysys</groupId>
<artifactId>escheduler</artifactId> <artifactId>escheduler</artifactId>
<version>1.0.4-SNAPSHOT</version> <version>1.1.0-SNAPSHOT</version>
<packaging>pom</packaging> <packaging>pom</packaging>
<name>escheduler</name> <name>escheduler</name>
<url>http://maven.apache.org</url> <url>http://maven.apache.org</url>

1
sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql

@ -1 +0,0 @@
INSERT INTO `t_escheduler_version` (`version`) VALUES ('1.1.0');
Loading…
Cancel
Save