diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index ed3baa40f0..be8b5fbb6d 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.0.4-SNAPSHOT + 1.1.0-SNAPSHOT escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index 0c2a3019cd..b0d92d54a8 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -1,10 +1,9 @@ - + 4.0.0 cn.analysys escheduler - 1.0.4-SNAPSHOT + 1.1.0-SNAPSHOT escheduler-api jar @@ -48,6 +47,10 @@ org.springframework.boot spring-boot-starter-tomcat + + log4j-to-slf4j + org.apache.logging.log4j + diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index 8e337a3e61..35b025b1d8 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.0.4-SNAPSHOT + 1.1.0-SNAPSHOT escheduler-common escheduler-common diff --git a/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java b/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java index e6c9df2833..eed7fb1ac5 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java @@ -151,7 +151,27 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue { int size = list.size(); - Set taskTreeSet = new TreeSet<>(); + Set taskTreeSet = new TreeSet<>(new Comparator() { + @Override + public int compare(String o1, String o2) { + + String s1 = o1; + String s2 = o2; + String[] s1Array = s1.split(Constants.UNDERLINE); + if(s1Array.length>4){ + // warning: if this length > 5, need to be changed + s1 = s1.substring(0, s1.lastIndexOf(Constants.UNDERLINE) ); + } + + String[] s2Array = s2.split(Constants.UNDERLINE); + if(s2Array.length>4){ + // warning: if this length > 5, need to be changed + s2 = s2.substring(0, s2.lastIndexOf(Constants.UNDERLINE) ); + } + + return s1.compareTo(s2); + } + }); for (int i = 0; i < size; i++) { @@ -173,8 +193,8 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue { continue; } } + formatTask += Constants.UNDERLINE + taskDetailArrs[4]; } - taskTreeSet.add(formatTask); } @@ -229,7 +249,7 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue { int taskId = Integer.parseInt(taskArray[3]); StringBuilder sb = new StringBuilder(50); - String destTask = String.format("%s_%s_%s_%s", taskArray[0], processInstanceId, taskArray[3], taskId); + String destTask = String.format("%s_%s_%s_%s", taskArray[0], processInstanceId, taskArray[2], taskId); sb.append(destTask); diff --git a/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java b/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java index 72a6e46200..21d2f5858e 100644 --- a/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java +++ b/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java @@ -17,6 +17,8 @@ package cn.escheduler.common.queue; import cn.escheduler.common.Constants; +import cn.escheduler.common.utils.IpUtils; +import cn.escheduler.common.utils.OSUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -58,31 +60,31 @@ public class TaskQueueImplTest { @Test public void testAdd(){ + //add - tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_1_1_1_2130706433,3232236775"); + tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_0_1_1_-1"); tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"0_1_1_1_2130706433,3232236775"); - tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_1_0_1_2130706433,3232236775"); + tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_1_0_1_2130706433,3232236775,"+IpUtils.ipToLong(OSUtils.getHost())); tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_2_1_1_2130706433,3232236775"); List tasks = tasksQueue.poll(Constants.SCHEDULER_TASKS_QUEUE, 1); - if(tasks.size() < 0){ + if(tasks.size() <= 0){ return; } //pop String node1 = tasks.get(0); - assertEquals(node1,"0_0000000001_1_0000000001"); + assertEquals(node1,"1_0_1_1_-1"); tasks = tasksQueue.poll(Constants.SCHEDULER_TASKS_QUEUE, 1); - if(tasks.size() < 0){ + if(tasks.size() <= 0){ return; } String node2 = tasks.get(0); - assertEquals(node2,"0_0000000001_1_0000000001"); } diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index 2f2a5ad74e..4d8fb6912e 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.0.4-SNAPSHOT + 1.1.0-SNAPSHOT escheduler-dao escheduler-dao @@ -37,6 +37,10 @@ org.apache.tomcat tomcat-jdbc + + log4j-to-slf4j + org.apache.logging.log4j + diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index 4ec4ea2260..e3d7c91a92 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.0.4-SNAPSHOT + 1.1.0-SNAPSHOT 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index 6341539e4c..7f149278c8 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.0.4-SNAPSHOT + 1.1.0-SNAPSHOT escheduler-server escheduler-server diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java index 94caffe596..e1df0b71a8 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java @@ -18,6 +18,7 @@ package cn.escheduler.server.worker.task; import cn.escheduler.common.Constants; import cn.escheduler.common.utils.FileUtils; +import cn.escheduler.common.utils.PropertyUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,11 +72,11 @@ public class PythonCommandExecutor extends AbstractCommandExecutor { if (!Files.exists(Paths.get(commandFile))) { logger.info("generate command file:{}", commandFile); - StringBuilder sb = new StringBuilder(200); + StringBuilder sb = new StringBuilder(); sb.append("#-*- encoding=utf8 -*-\n"); sb.append("\n\n"); - sb.append(String.format("import py_%s_node\n",taskAppId)); + sb.append(execCommand); logger.info(sb.toString()); // write data to file @@ -86,8 +87,8 @@ public class PythonCommandExecutor extends AbstractCommandExecutor { @Override protected String commandType() { - String envPath = System.getProperty("user.dir") + Constants.SINGLE_SLASH + "conf"+ - Constants.SINGLE_SLASH +"env" + Constants.SINGLE_SLASH + Constants.ESCHEDULER_ENV_SH; + String envPath = PropertyUtils.getString(Constants.ESCHEDULER_ENV_PATH); + String pythonHome = getPythonHome(envPath); if (StringUtils.isEmpty(pythonHome)){ return PYTHON; diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java index 49d754404a..c446215a38 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java @@ -112,14 +112,14 @@ public class PythonTask extends AbstractTask { */ private String buildCommand() throws Exception { // generate scripts - String fileName = String.format("%s/py_%s_node.py", taskDir, taskProps.getTaskAppId()); - Path path = new File(fileName).toPath(); +// String fileName = String.format("%s/py_%s_node.py", taskDir, taskProps.getTaskAppId()); +// Path path = new File(fileName).toPath(); - if (Files.exists(path)) { - return fileName; - } +// if (Files.exists(path)) { +// return fileName; +// } String rawScript = pythonParameters.getRawScript().replaceAll("\\r\\n", "\n"); @@ -140,19 +140,20 @@ public class PythonTask extends AbstractTask { } - pythonParameters.setRawScript(rawScript); +// pythonParameters.setRawScript(rawScript); logger.info("raw script : {}", pythonParameters.getRawScript()); logger.info("task dir : {}", taskDir); - Set perms = PosixFilePermissions.fromString("rwxr-xr-x"); - FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); - - Files.createFile(path, attr); - - Files.write(path, pythonParameters.getRawScript().getBytes(), StandardOpenOption.APPEND); - - return fileName; +// Set perms = PosixFilePermissions.fromString("rwxr-xr-x"); +// FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); +// +// Files.createFile(path, attr); +// +// Files.write(path, pythonParameters.getRawScript().getBytes(), StandardOpenOption.APPEND); +// +// return fileName; + return rawScript; } @Override diff --git a/escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue b/escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue index e6302fdd1f..a552e4cc00 100644 --- a/escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue +++ b/escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue @@ -16,7 +16,7 @@
- {{$t('failure command number')}}} + {{$t('failure command number')}}
{{commandCountData.errorCount}} diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue index a61f0634d9..42bb7905a0 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue @@ -254,11 +254,6 @@ this.store.dispatch(api, searchParams).then(res => { this.previewTimes = res - if (this.previewTimes.length) { - resolve() - } else { - reject(new Error(0)) - } }) } }, diff --git a/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue b/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue index 574a995ec4..378f410d38 100644 --- a/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue +++ b/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue @@ -131,7 +131,8 @@ } }, _verification () { - let regEmail = /^([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+@([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+\.[a-zA-Z]{2,3}$/ // eslint-disable-line + let regEmail = /^([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+@([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+\.[a-zA-Z]{2,}$/ // eslint-disable-line + // Mobile phone number regular let regPhone = /^1(3|4|5|6|7|8)\d{9}$/; // eslint-disable-line @@ -184,7 +185,10 @@ _getTenantList () { return new Promise((resolve, reject) => { this.store.dispatch('security/getTenantList').then(res => { - this.tenantList = _.map(res, v => { + let arr = _.filter(res, (o) => { + return o.id !== -1 + }) + this.tenantList = _.map(arr, v => { return { id: v.id, code: v.tenantName diff --git a/install.sh b/install.sh index d58be482fd..33acdf5eec 100644 --- a/install.sh +++ b/install.sh @@ -144,6 +144,7 @@ singleYarnIp="ark1" hdfsPath="/escheduler" # 拥有在hdfs根路径/下创建目录权限的用户 +# 注意：如果开启了kerberos，则直接hdfsRootUser=，就可以 hdfsRootUser="hdfs" # common 配置 diff --git a/pom.xml b/pom.xml index e4186d9b81..a97650aafb 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.0.4-SNAPSHOT + 1.1.0-SNAPSHOT pom escheduler http://maven.apache.org diff --git a/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql b/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql index 4ddd9ddcdc..e69de29bb2 100644 --- a/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql +++ b/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql @@ -1 +0,0 @@ -INSERT INTO `t_escheduler_version` (`version`) VALUES ('1.1.0'); \ No newline at end of file