From b3dae1930d34eed0c0c278536a5841ae12bf1a2a Mon Sep 17 00:00:00 2001 From: lidongdai Date: Thu, 26 Sep 2019 18:45:02 +0800 Subject: [PATCH 01/43] update README about DolphinScheduler --- README.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 51e0cbb8bc..6352bd5f10 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -Easy Scheduler +Dolphin Scheduler ============ [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) -[![Total Lines](https://tokei.rs/b1/github/analysys/EasyScheduler?category=lines)](https://github.com/analysys/EasyScheduler) +[![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler) -> Easy Scheduler for Big Data +> Dolphin Scheduler for Big Data -[![Stargazers over time](https://starchart.cc/analysys/EasyScheduler.svg)](https://starchart.cc/analysys/EasyScheduler) +[![Stargazers over time](https://starchart.cc/apache/incubator-dolphinscheduler.svg)](https://starchart.cc/apache/incubator-dolphinscheduler) [![EN doc](https://img.shields.io/badge/document-English-blue.svg)](README.md) [![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](README_zh_CN.md) @@ -32,12 +32,12 @@ Its main objectives are as follows: - There are more waiting partners to explore -### What's in Easy Scheduler +### What's in Dolphin Scheduler Stability | Easy to use | Features | Scalability | -- | -- | -- | -- Decentralized multi-master and multi-worker | Visualization process defines key information such as task status, task type, retry times, task running machine, visual variables and so on at a glance.  |  Support pause, recover operation | support custom task types -HA is supported by itself | All process definition operations are visualized, dragging tasks to draw DAGs, configuring data sources and resources. At the same time, for third-party systems, the api mode operation is provided. | Users on easyscheduler can achieve many-to-one or one-to-one mapping relationship through tenants and Hadoop users, which is very important for scheduling large data jobs. " | The scheduler uses distributed scheduling, and the overall scheduling capability will increase linearly with the scale of the cluster. Master and Worker support dynamic online and offline. +HA is supported by itself | All process definition operations are visualized, dragging tasks to draw DAGs, configuring data sources and resources. At the same time, for third-party systems, the api mode operation is provided. | Users on DolphinScheduler can achieve many-to-one or one-to-one mapping relationship through tenants and Hadoop users, which is very important for scheduling large data jobs. " | The scheduler uses distributed scheduling, and the overall scheduling capability will increase linearly with the scale of the cluster. Master and Worker support dynamic online and offline. Overload processing: Task queue mechanism, the number of schedulable tasks on a single machine can be flexibly configured, when too many tasks will be cached in the task queue, will not cause machine jam. | One-click deployment | Supports traditional shell tasks, and also support big data platform task scheduling: MR, Spark, SQL (mysql, postgresql, hive, sparksql), Python, Procedure, Sub_Process | | @@ -54,30 +54,30 @@ Overload processing: Task queue mechanism, the number of schedulable tasks on a ### Document -- Backend deployment documentation +- Backend deployment documentation -- Front-end deployment documentation +- Front-end deployment documentation -- [**User manual**](https://analysys.github.io/easyscheduler_docs_cn/系统使用手册.html?_blank "User manual") +- [**User manual**](https://dolphinscheduler.apache.org/en-us/docs/user_doc/system-manual.html?_blank "System manual") -- [**Upgrade document**](https://analysys.github.io/easyscheduler_docs_cn/升级文档.html?_blank "Upgrade document") +- [**Upgrade document**](https://dolphinscheduler.apache.org/en-us/docs/release/upgrade.html?_blank "Upgrade document") - Online Demo -More documentation please refer to [EasyScheduler online documentation] +More documentation please refer to [DolphinScheduler online documentation] ### Recent R&D plan -Work plan of Easy Scheduler: [R&D plan](https://github.com/analysys/EasyScheduler/projects/1), where `In Develop` card is the features of 1.1.0 version , TODO card is to be done (including feature ideas) +Work plan of Dolphin Scheduler: [R&D plan](https://github.com/apache/incubator-dolphinscheduler/projects/1), Under the `In Develop` card is what is currently being developed, TODO card is to be done (including feature ideas) ### How to contribute code Welcome to participate in contributing code, please refer to the process of submitting the code: -[[How to contribute code](https://github.com/analysys/EasyScheduler/issues/310)] +[[How to contribute code](https://github.com/apache/incubator-dolphinscheduler/issues/310)] ### Thanks -Easy Scheduler uses a lot of excellent open source projects, such as google guava, guice, grpc, netty, ali bonecp, quartz, and many open source projects of apache, etc. -It is because of the shoulders of these open source projects that the birth of the Easy Scheduler is possible. We are very grateful for all the open source software used! We also hope that we will not only be the beneficiaries of open source, but also be open source contributors, so we decided to contribute to easy scheduling and promised long-term updates. We also hope that partners who have the same passion and conviction for open source will join in and contribute to open source! +Dolphin Scheduler uses a lot of excellent open source projects, such as google guava, guice, grpc, netty, ali bonecp, quartz, and many open source projects of apache, etc. +It is because of the shoulders of these open source projects that the birth of the Dolphin Scheduler is possible. We are very grateful for all the open source software used! We also hope that we will not only be the beneficiaries of open source, but also be open source contributors. We also hope that partners who have the same passion and conviction for open source will join in and contribute to open source! ### Get Help 1. Submit an issue @@ -85,7 +85,7 @@ It is because of the shoulders of these open source projects that the birth of t 1. Contact WeChat group manager, ID 510570367. This is for Mandarin(CN) discussion. ### License -Please refer to [LICENSE](https://github.com/analysys/EasyScheduler/blob/dev/LICENSE) file. +Please refer to [LICENSE](https://github.com/apache/incubator-dolphinscheduler/blob/dev/LICENSE) file. From fb786d8db77ee003eb3eef61ad7f39a151306e73 Mon Sep 17 00:00:00 2001 From: dailidong Date: Fri, 1 Nov 2019 09:28:48 +0800 Subject: [PATCH 02/43] Update issue templates --- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- .github/ISSUE_TEMPLATE/question.md | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 .github/ISSUE_TEMPLATE/question.md diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 8cd481d442..f9acdb30be 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -2,7 +2,7 @@ name: Feature request about: Suggest an idea for this project title: "[Feature]" -labels: new feature +labels: '' assignees: '' --- diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 0000000000..59c8ce7716 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,23 @@ +--- +name: question +about: have a question wanted to be help +title: "[QUESTION] question title" +labels: question +assignees: '' + +--- + +*For better global communication, please give priority to using English description, thx! * + +**Describe the question** +A clear and concise description of what the question is. + + +**Which version of DolphinScheduler:** + -[1.1.0-preview] + +**Additional context** +Add any other context about the problem here. + +**Requirement or improvement +- Please describe about your requirements or improvement suggestions. From 608674900b5de2a311fc6e854602a75e93f600d4 Mon Sep 17 00:00:00 2001 From: dailidong Date: Wed, 25 Dec 2019 20:00:56 +0800 Subject: [PATCH 03/43] update --- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index f9acdb30be..8cd481d442 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -2,7 +2,7 @@ name: Feature request about: Suggest an idea for this project title: "[Feature]" -labels: '' +labels: new feature assignees: '' --- From 5250ab0df4778f93c169a4bfac53e0d393ae2431 Mon Sep 17 00:00:00 2001 From: dailidong Date: Wed, 25 Dec 2019 20:11:52 +0800 Subject: [PATCH 04/43] regularize api pom xml update rpc maven compile to 1.8 --- dolphinscheduler-api/pom.xml | 87 +++++++++++++++++++++--------------- dolphinscheduler-rpc/pom.xml | 4 +- 2 files changed, 54 insertions(+), 37 deletions(-) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 06a444aa6d..986c1904d1 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -30,14 +30,17 @@ 5.5.23 2.5 + 1.9.3 + 2.9.2 - - org.apache.dolphinscheduler - dolphinscheduler-alert - - + + org.apache.dolphinscheduler + dolphinscheduler-alert + + + org.apache.dolphinscheduler dolphinscheduler-server @@ -96,25 +99,8 @@ - - org.springframework.boot - spring-boot-starter-test - test - - - org.ow2.asm - asm - - - org.springframework.boot - spring-boot - - - org.springframework.boot - spring-boot-autoconfigure - - - + + org.springframework.boot spring-boot-starter-aop @@ -125,6 +111,7 @@ + org.springframework spring-context @@ -134,6 +121,7 @@ org.apache.httpcomponents httpcore + org.apache.httpcomponents httpclient @@ -143,10 +131,12 @@ com.fasterxml.jackson.core jackson-annotations + com.fasterxml.jackson.core jackson-databind + com.fasterxml.jackson.core jackson-core @@ -181,19 +171,19 @@ io.springfox springfox-swagger2 - 2.9.2 + ${springfox.version} io.springfox springfox-swagger-ui - 2.9.2 + ${springfox.version} com.github.xiaoymin swagger-bootstrap-ui - 1.9.3 + ${swagger.version} @@ -201,38 +191,37 @@ dolphinscheduler-rpc - - junit - junit - 4.12 - test - - org.apache.curator curator-framework + org.apache.curator curator-recipes + org.apache.hadoop hadoop-common + org.apache.hadoop hadoop-client + org.apache.hadoop hadoop-hdfs + org.apache.hadoop hadoop-yarn-common + org.apache.hadoop hadoop-aws @@ -250,5 +239,33 @@ ${servlet-api.version} + + + org.springframework.boot + spring-boot-starter-test + test + + + org.ow2.asm + asm + + + org.springframework.boot + spring-boot + + + org.springframework.boot + spring-boot-autoconfigure + + + + + + junit + junit + 4.12 + test + + - \ No newline at end of file + diff --git a/dolphinscheduler-rpc/pom.xml b/dolphinscheduler-rpc/pom.xml index f0ac1a0e43..680a4a24c0 100644 --- a/dolphinscheduler-rpc/pom.xml +++ b/dolphinscheduler-rpc/pom.xml @@ -31,8 +31,8 @@ UTF-8 - 1.7 - 1.7 + 1.8 + 1.8 3.5.1 1.9.0 From dcc2ff690f89958bb5092e7d5159aa2ff2a106de Mon Sep 17 00:00:00 2001 From: dailidong Date: Wed, 25 Dec 2019 20:19:52 +0800 Subject: [PATCH 05/43] regularize api pom xml --- dolphinscheduler-api/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 986c1904d1..307adda12a 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -263,7 +263,6 @@ junit junit - 4.12 test From 9169a7cc4059f0305e4d6facf8790e303fc0fd40 Mon Sep 17 00:00:00 2001 From: dailidong Date: Tue, 7 Jan 2020 17:11:44 +0800 Subject: [PATCH 06/43] change commons.lang3.StringUtils to common.utils.StringUtils --- .../apache/dolphinscheduler/api/security/SecurityConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java index f2cf72cdbb..8e1f1ac1f8 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.api.security; -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; From 7c1b37ae206ed8c9ab720dadf2a8e5789619984b Mon Sep 17 00:00:00 2001 From: dailidong Date: Tue, 7 Jan 2020 17:16:14 +0800 Subject: [PATCH 07/43] update pom.xml --- dolphinscheduler-api/pom.xml | 37 ------------------------------------ 1 file changed, 37 deletions(-) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index a1141b5726..ae28a48bb5 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -27,13 +27,6 @@ ${project.artifactId} jar - - 5.5.23 - 2.5 - 1.9.3 - 2.9.2 - - org.apache.dolphinscheduler @@ -117,31 +110,6 @@ spring-context - - org.apache.httpcomponents - httpcore - - - - org.apache.httpcomponents - httpclient - - - - com.fasterxml.jackson.core - jackson-annotations - - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.core - jackson-core - - com.alibaba fastjson @@ -171,19 +139,16 @@ io.springfox springfox-swagger2 - ${springfox.version} io.springfox springfox-swagger-ui - ${springfox.version} com.github.xiaoymin swagger-bootstrap-ui - ${swagger.version} @@ -230,13 +195,11 @@ tomcat jasper-runtime - ${jasper-runtime.version} javax.servlet servlet-api - ${servlet-api.version} From ce330ca0efb7097911ae2bd3cdc0b0a08e1a2329 Mon Sep 17 00:00:00 2001 From: dailidong Date: Tue, 7 Jan 2020 17:46:42 +0800 Subject: [PATCH 08/43] update --- README.md | 16 ++++++------- README_zh_CN.md | 24 +++++++++---------- .../server/master/MasterServer.java | 6 ++--- .../runner/MasterBaseTaskExecThread.java | 2 +- .../master/runner/MasterExecThread.java | 8 +++---- .../master/runner/MasterSchedulerThread.java | 2 +- .../master/runner/MasterTaskExecThread.java | 2 +- .../quartz/DruidConnectionProvider.java | 4 ++-- .../server/quartz/QuartzExecutors.java | 3 +-- .../server/rpc/LogClient.java | 6 ++--- .../server/utils/ProcessUtils.java | 4 ++-- .../server/worker/runner/FetchTaskThread.java | 9 ++++--- 12 files changed, 42 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index bc7dba684f..93fb7a40b7 100644 --- a/README.md +++ b/README.md @@ -58,25 +58,25 @@ Overload processing: Task queue mechanism, the number of schedulable tasks on a ### Document -- Backend deployment documentation +- Backend deployment documentation -- Front-end deployment documentation +- Front-end deployment documentation -- [**User manual**](https://dolphinscheduler.apache.org/en-us/docs/user_doc/system-manual.html?_blank "System manual") +- [**User manual**](https://dolphinscheduler.apache.org/en-us/docs/1.2.0/user_doc/system-manual.html?_blank "System manual") -- [**Upgrade document**](https://dolphinscheduler.apache.org/en-us/docs/release/upgrade.html?_blank "Upgrade document") +- [**Upgrade document**](https://dolphinscheduler.apache.org/en-us/docs/1.2.0/release/upgrade.html?_blank "Upgrade document") - Online Demo -More documentation please refer to [DolphinScheduler online documentation] +More documentation please refer to [DolphinScheduler online documentation] ### Recent R&D plan Work plan of Dolphin Scheduler: [R&D plan](https://github.com/apache/incubator-dolphinscheduler/projects/1), Under the `In Develop` card is what is currently being developed, TODO card is to be done (including feature ideas) -### How to contribute code +### How to contribute -Welcome to participate in contributing code, please refer to the process of submitting the code: -[[How to contribute code](https://github.com/apache/incubator-dolphinscheduler/issues/310)] +Welcome to participate in contributing, please refer to the process of submitting the code: +[[How to contribute](https://dolphinscheduler.apache.org/en-us/docs/development/contribute.html)] ### How to Build diff --git a/README_zh_CN.md b/README_zh_CN.md index 8141763c33..c7173ad9ba 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -44,27 +44,27 @@ Dolphin Scheduler Official Website ### 文档 -- 后端部署文档 +- 后端部署文档 -- 前端部署文档 +- 前端部署文档 -- [**使用手册**](https://dolphinscheduler.apache.org/zh-cn/docs/user_doc/system-manual.html?_blank "系统使用手册") +- [**使用手册**](https://dolphinscheduler.apache.org/zh-cn/docs/1.2.0/user_doc/system-manual.html?_blank "系统使用手册") -- [**升级文档**](https://dolphinscheduler.apache.org/zh-cn/docs/release/upgrade.html?_blank "升级文档") +- [**升级文档**](https://dolphinscheduler.apache.org/zh-cn/docs/1.2.0/release/upgrade.html?_blank "升级文档") - 我要体验 -更多文档请参考 DolphinScheduler中文在线文档 +更多文档请参考 DolphinScheduler中文在线文档 ### 近期研发计划 DolphinScheduler的工作计划：研发计划 ，其中 In Develop卡片下是正在研发的功能，TODO卡片是待做事项(包括 feature ideas) -### 贡献代码 +### 参与贡献 -非常欢迎大家来参与贡献代码，提交代码流程请参考： -[[How to contribute code](https://github.com/apache/incubator-dolphinscheduler/issues/310)] +非常欢迎大家来参与贡献，贡献流程请参考： +[[参与贡献](https://dolphinscheduler.apache.org/zh-cn/docs/development/contribute.html)] ### How to Build @@ -88,12 +88,12 @@ Dolphin Scheduler使用了很多优秀的开源项目，比如google的guava、g ### 获得帮助 -1. Submit an issue -1. Mail to dev-subscribe@dolphinscheduler.apache.org, follow the reply to subscribe the mail list. then you can send mail to dev@dolphinscheduler.apache.org. -1. Contact WeChat group manager, ID 510570367. This is for Mandarin(CN) discussion. +1. 提交issue +1. 先订阅邮件开发列表:[订阅邮件列表](https://dolphinscheduler.apache.org/zh-cn/docs/1.2.0/user_doc/subscribe.html), 订阅成功后发送邮件到dev@dolphinscheduler.apache.org. +1. 联系微信群助手(ID:dailidong66). 微信仅用于中国用户讨论. ### 版权 -Please refer to [LICENSE](https://github.com/apache/incubator-dolphinscheduler/blob/dev/LICENSE) file. +请参考 [LICENSE](https://github.com/apache/incubator-dolphinscheduler/blob/dev/LICENSE) 文件. diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index c4cec68683..e5b9daf6f0 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -141,7 +141,7 @@ public class MasterServer implements IStoppable { } catch (SchedulerException e1) { logger.error("QuartzExecutors shutdown failed : " + e1.getMessage(), e1); } - logger.error("start Quartz failed : " + e.getMessage(), e); + logger.error("start Quartz failed", e); } @@ -183,7 +183,7 @@ public class MasterServer implements IStoppable { //thread sleep 3 seconds for thread quitely stop Thread.sleep(3000L); }catch (Exception e){ - logger.warn("thread sleep exception:" + e.getMessage(), e); + logger.warn("thread sleep exception ", e); } try { heartbeatMasterService.shutdownNow(); @@ -228,7 +228,7 @@ public class MasterServer implements IStoppable { } catch (Exception e) { - logger.error("master server stop exception : " + e.getMessage(), e); + logger.error("master server stop exception ", e); System.exit(-1); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java index 5c96757072..c1552c4621 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java @@ -140,7 +140,7 @@ public class MasterBaseTaskExecThread implements Callable { } Thread.sleep(commitRetryInterval); } catch (Exception e) { - logger.error("task commit to mysql and queue failed : " + e.getMessage(),e); + logger.error("task commit to mysql and queue failed",e); } retryTimes += 1; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index ade4738884..a91f8c17e6 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -170,7 +170,7 @@ public class MasterExecThread implements Runnable { executeProcess(); } }catch (Exception e){ - logger.error("master exec thread exception: " + e.getMessage(), e); + logger.error("master exec thread exception", e); logger.error("process execute failed, process id:{}", processInstance.getId()); processInstance.setState(ExecutionStatus.FAILURE); processInstance.setEndTime(new Date()); @@ -355,7 +355,7 @@ public class MasterExecThread implements Runnable { try { FileUtils.deleteDirectory(new File(execLocalPath)); } catch (IOException e) { - logger.error("delete exec dir failed : " + e.getMessage(), e); + logger.error("delete exec dir failed ", e); } } } @@ -733,7 +733,7 @@ public class MasterExecThread implements Runnable { Date endTime = DateUtils.getScheduleDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_END_DATE)); return processInstance.getScheduleTime().equals(endTime); } catch (Exception e) { - logger.error("complement end failed : " + e.getMessage(),e); + logger.error("complement end failed ",e); return false; } } @@ -1005,7 +1005,7 @@ public class MasterExecThread implements Runnable { return task; } }catch (Exception e){ - logger.error("get recovery task instance failed : " + e.getMessage(),e); + logger.error("get recovery task instance failed ",e); } return null; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java index 5f594b3fa0..1d34bbd147 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java @@ -134,7 +134,7 @@ public class MasterSchedulerThread implements Runnable { } } }catch (Exception e){ - logger.error("master scheduler thread exception : " + e.getMessage(),e); + logger.error("master scheduler thread exception",e); }finally{ AbstractZKClient.releaseMutex(mutex); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java index 7d10591e0d..1861e8a908 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java @@ -137,7 +137,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { processInstance = processDao.findProcessInstanceById(processInstance.getId()); Thread.sleep(Constants.SLEEP_TIME_MILLIS); } catch (Exception e) { - logger.error("exception: "+ e.getMessage(),e); + logger.error("exception",e); if (processInstance != null) { logger.error("wait task quit failed, instance id:{}, task id:{}", processInstance.getId(), taskInstance.getId()); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/quartz/DruidConnectionProvider.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/quartz/DruidConnectionProvider.java index 32932127f0..05100ac374 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/quartz/DruidConnectionProvider.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/quartz/DruidConnectionProvider.java @@ -18,9 +18,9 @@ package org.apache.dolphinscheduler.server.quartz; import com.alibaba.druid.pool.DruidDataSource; import org.quartz.SchedulerException; +import org.quartz.utils.ConnectionProvider; import java.sql.Connection; import java.sql.SQLException; -import org.quartz.utils.ConnectionProvider; /** * druid connection provider @@ -137,7 +137,7 @@ public class DruidConnectionProvider implements ConnectionProvider { datasource.setDriverClassName(this.driver); } catch (Exception e) { try { - throw new SchedulerException("Problem setting driver class name on datasource: " + e.getMessage(), e); + throw new SchedulerException("Problem setting driver class name on datasource", e); } catch (SchedulerException e1) { } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/quartz/QuartzExecutors.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/quartz/QuartzExecutors.java index bb0f79897d..d4ad08faec 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/quartz/QuartzExecutors.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/quartz/QuartzExecutors.java @@ -27,7 +27,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; -import java.util.Calendar; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -198,7 +197,7 @@ public class QuartzExecutors { } catch (Exception e) { logger.error("add job failed", e); - throw new RuntimeException("add job failed:"+e.getMessage()); + throw new RuntimeException("add job failed", e); } finally { lock.writeLock().unlock(); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LogClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LogClient.java index 3f12ae75dd..1c6c97b88f 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LogClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LogClient.java @@ -102,7 +102,7 @@ public class LogClient { retStrInfo = blockingStub.rollViewLog(pathParameter); return retStrInfo.getMsg(); } catch (StatusRuntimeException e) { - logger.error("roll view log failed : " + e.getMessage(), e); + logger.error("roll view log failed", e); return null; } } @@ -122,7 +122,7 @@ public class LogClient { retStrInfo = blockingStub.viewLog(pathParameter); return retStrInfo.getMsg(); } catch (StatusRuntimeException e) { - logger.error("view log failed : " + e.getMessage(), e); + logger.error("view log failed", e); return null; } } @@ -142,7 +142,7 @@ public class LogClient { retByteInfo = blockingStub.getLogBytes(pathParameter); return retByteInfo.getData().toByteArray(); } catch (StatusRuntimeException e) { - logger.error("get log bytes failed : " + e.getMessage(), e); + logger.error("get log bytes failed ", e); return null; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java index 7cba139cec..0b621a9bb0 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java @@ -308,7 +308,7 @@ public class ProcessUtils { Runtime.getRuntime().exec(runCmd); } catch (Exception e) { - logger.error("kill application failed : " + e.getMessage(), e); + logger.error("kill application failed", e); } } } @@ -337,7 +337,7 @@ public class ProcessUtils { killYarnJob(taskInstance); } catch (Exception e) { - logger.error("kill failed : " + e.getMessage(), e); + logger.error("kill task failed", e); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java index 60cfd82cdd..536fd0b0ca 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java @@ -186,6 +186,10 @@ public class FetchTaskThread implements Runnable{ continue; } + if(!checkWorkerGroup(taskInstance, OSUtils.getHost())){ + continue; + } + Tenant tenant = processDao.getTenantForProcess(taskInstance.getProcessInstance().getTenantId(), taskInstance.getProcessDefine().getUserId()); @@ -203,11 +207,6 @@ public class FetchTaskThread implements Runnable{ logger.info("worker fetch taskId : {} from queue ", taskInstId); - - if(!checkWorkerGroup(taskInstance, OSUtils.getHost())){ - continue; - } - // local execute path String execLocalPath = getExecLocalPath(); From f9afc0a475ceb97f2ad3bfeffefdf738fb311058 Mon Sep 17 00:00:00 2001 From: dailidong Date: Fri, 17 Jan 2020 21:15:47 +0800 Subject: [PATCH 09/43] correct equals method --- .../org/apache/dolphinscheduler/common/model/DateInterval.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/DateInterval.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/DateInterval.java index 9dc2f34cc1..6ce0bdb6b6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/DateInterval.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/DateInterval.java @@ -34,7 +34,7 @@ public class DateInterval { } @Override - public boolean equals(Object obj) { + public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; DateInterval that = (DateInterval) o; From 39e98b27ec9b611f3602c8b6c23dc82927c042a6 Mon Sep 17 00:00:00 2001 From: dailidong Date: Tue, 4 Feb 2020 23:42:16 +0800 Subject: [PATCH 10/43] jasper-runtime is needed when api server start --- dolphinscheduler-api/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 4dd2f31f8e..cf2aadfcb4 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -178,6 +178,11 @@ hadoop-aws + + tomcat + jasper-runtime + + javax.servlet servlet-api From db45500e6f5b34a6245cbf9c80bf51dbcc40ae26 Mon Sep 17 00:00:00 2001 From: dailidong Date: Tue, 4 Feb 2020 23:54:35 +0800 Subject: [PATCH 11/43] jasper-runtime jar is needed when api server start --- dolphinscheduler-api/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index cf2aadfcb4..35a6f0616a 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -27,6 +27,10 @@ ${project.artifactId} jar + + 5.5.23 + + org.apache.dolphinscheduler @@ -181,6 +185,7 @@ tomcat jasper-runtime + ${jasper-runtime.version} From 835383105c545754652789e95bab54dc0b88d61f Mon Sep 17 00:00:00 2001 From: dailidong Date: Wed, 5 Feb 2020 20:56:47 +0800 Subject: [PATCH 12/43] combine logback config of master/worker/alert/api server to one logback.xml --- .../conf/combined_logback.xml | 6 +- .../dolphinscheduler/conf/worker_logback.xml | 6 +- .../src/main/resources/alert_logback.xml | 49 ----- .../src/main/resources/apiserver_logback.xml | 60 ------- .../main/resources/application-api.properties | 2 - dolphinscheduler-common/pom.xml | 7 + .../common}/log/MasterLogFilter.java | 2 +- .../common}/log/SensitiveDataConverter.java | 6 +- .../common}/log/TaskLogDiscriminator.java | 4 +- .../common}/log/TaskLogFilter.java | 4 +- .../common}/log/WorkerLogFilter.java | 6 +- .../common}/utils/LoggerUtils.java | 9 +- .../common/utils/SensitiveLogUtils.java | 4 +- .../src/main/resources/logback.xml | 169 ++++++++++++++++++ .../log/SensitiveDataConverterTest.java | 6 +- .../common/utils/SensitiveLogUtilsTest.java | 9 +- .../server/utils/ProcessUtils.java | 1 + .../worker/runner/TaskScheduleThread.java | 4 +- .../worker/task/AbstractCommandExecutor.java | 2 +- .../src/main/resources/master_logback.xml | 52 ------ .../src/main/resources/worker_logback.xml | 81 --------- .../shell/ShellCommandExecutorTest.java | 2 +- .../server/worker/sql/SqlExecutorTest.java | 2 +- script/dolphinscheduler-daemon.sh | 12 +- 24 files changed, 224 insertions(+), 281 deletions(-) delete mode 100644 dolphinscheduler-alert/src/main/resources/alert_logback.xml delete mode 100644 dolphinscheduler-api/src/main/resources/apiserver_logback.xml rename {dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master => dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common}/log/MasterLogFilter.java (96%) rename {dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker => dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common}/log/SensitiveDataConverter.java (92%) rename {dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker => dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common}/log/TaskLogDiscriminator.java (94%) rename {dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker => dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common}/log/TaskLogFilter.java (93%) rename {dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker => dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common}/log/WorkerLogFilter.java (91%) rename {dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server => dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common}/utils/LoggerUtils.java (91%) rename dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtil.java => dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java (93%) create mode 100644 dolphinscheduler-common/src/main/resources/logback.xml rename {dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker => dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common}/log/SensitiveDataConverterTest.java (93%) rename dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtilTest.java => dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtilsTest.java (79%) delete mode 100644 dolphinscheduler-server/src/main/resources/master_logback.xml delete mode 100644 dolphinscheduler-server/src/main/resources/worker_logback.xml diff --git a/dockerfile/conf/dolphinscheduler/conf/combined_logback.xml b/dockerfile/conf/dolphinscheduler/conf/combined_logback.xml index 6bdb97cf00..7a9a5b4621 100644 --- a/dockerfile/conf/dolphinscheduler/conf/combined_logback.xml +++ b/dockerfile/conf/dolphinscheduler/conf/combined_logback.xml @@ -31,8 +31,8 @@ INFO - - + + taskAppId ${log.base} @@ -52,7 +52,7 @@ ${log.base}/dolphinscheduler-combined.log - + INFO diff --git a/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml b/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml index 9bbd9615c4..bf4a651e7c 100644 --- a/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml +++ b/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml @@ -31,8 +31,8 @@ INFO - - + + taskAppId ${log.base} @@ -52,7 +52,7 @@ ${log.base}/dolphinscheduler-worker.log - + INFO diff --git a/dolphinscheduler-alert/src/main/resources/alert_logback.xml b/dolphinscheduler-alert/src/main/resources/alert_logback.xml deleted file mode 100644 index 3474df8d2e..0000000000 --- a/dolphinscheduler-alert/src/main/resources/alert_logback.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - - - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - - - - ${log.base}/dolphinscheduler-alert.log - - ${log.base}/dolphinscheduler-alert.%d{yyyy-MM-dd_HH}.%i.log - 20 - 64MB - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - - - - - - \ No newline at end of file diff --git a/dolphinscheduler-api/src/main/resources/apiserver_logback.xml b/dolphinscheduler-api/src/main/resources/apiserver_logback.xml deleted file mode 100644 index 4c2ce0d2b3..0000000000 --- a/dolphinscheduler-api/src/main/resources/apiserver_logback.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - - - - - - - - - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - - - - - - INFO - - ${log.base}/dolphinscheduler-api-server.log - - ${log.base}/dolphinscheduler-api-server.%d{yyyy-MM-dd_HH}.%i.log - 168 - 64MB - - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - - - - - - - \ No newline at end of file diff --git a/dolphinscheduler-api/src/main/resources/application-api.properties b/dolphinscheduler-api/src/main/resources/application-api.properties index 6ec304a397..21f940baaf 100644 --- a/dolphinscheduler-api/src/main/resources/application-api.properties +++ b/dolphinscheduler-api/src/main/resources/application-api.properties @@ -15,8 +15,6 @@ # limitations under the License. # -logging.config=classpath:apiserver_logback.xml - # server port server.port=12345 diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index 955e956251..bd2448eee7 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -29,6 +29,7 @@ jar UTF-8 + 3.1.0 @@ -604,5 +605,11 @@ org.springframework spring-context + + + org.codehaus.janino + janino + ${codehaus.janino.version} + diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/log/MasterLogFilter.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/MasterLogFilter.java similarity index 96% rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/log/MasterLogFilter.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/MasterLogFilter.java index d6de484074..7b5d53a032 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/log/MasterLogFilter.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/MasterLogFilter.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.server.master.log; +package org.apache.dolphinscheduler.common.log; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.spi.ILoggingEvent; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/log/SensitiveDataConverter.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverter.java similarity index 92% rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/log/SensitiveDataConverter.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverter.java index 4a98e66c6c..971ce7149c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/log/SensitiveDataConverter.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverter.java @@ -14,14 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.server.worker.log; +package org.apache.dolphinscheduler.common.log; import ch.qos.logback.classic.pattern.MessageConverter; import ch.qos.logback.classic.spi.ILoggingEvent; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.SensitiveLogUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.server.utils.SensitiveLogUtil; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -77,7 +77,7 @@ public class SensitiveDataConverter extends MessageConverter { String password = matcher.group(); - String maskPassword = SensitiveLogUtil.maskDataSourcePwd(password); + String maskPassword = SensitiveLogUtils.maskDataSourcePwd(password); matcher.appendReplacement(sb, maskPassword); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/log/TaskLogDiscriminator.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminator.java similarity index 94% rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/log/TaskLogDiscriminator.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminator.java index fa00aed772..fd2b0766a8 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/log/TaskLogDiscriminator.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminator.java @@ -14,12 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.server.worker.log; +package org.apache.dolphinscheduler.common.log; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.sift.AbstractDiscriminator; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.server.utils.LoggerUtils; +import org.apache.dolphinscheduler.common.utils.LoggerUtils; /** * Task Log Discriminator diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/log/TaskLogFilter.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/TaskLogFilter.java similarity index 93% rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/log/TaskLogFilter.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/TaskLogFilter.java index 6398135481..ac258daf20 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/log/TaskLogFilter.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/TaskLogFilter.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.server.worker.log; +package org.apache.dolphinscheduler.common.log; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.filter.Filter; import ch.qos.logback.core.spi.FilterReply; -import org.apache.dolphinscheduler.server.utils.LoggerUtils; +import org.apache.dolphinscheduler.common.utils.LoggerUtils; /** * task log filter diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/log/WorkerLogFilter.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java similarity index 91% rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/log/WorkerLogFilter.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java index 23758f918a..abcc8bc619 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/log/WorkerLogFilter.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java @@ -14,12 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.server.worker.log; +package org.apache.dolphinscheduler.common.log; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.filter.Filter; import ch.qos.logback.core.spi.FilterReply; +import org.apache.dolphinscheduler.common.utils.LoggerUtils; + +import java.util.Arrays; /** * worker log filter @@ -40,6 +43,7 @@ public class WorkerLogFilter extends Filter { if (event.getThreadName().startsWith("Worker-")){ return FilterReply.ACCEPT; } + return FilterReply.DENY; } public void setLevel(String level) { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/LoggerUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java similarity index 91% rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/LoggerUtils.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java index 8b40d943c0..fc08eb645b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/LoggerUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.server.utils; +package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.Constants; import org.slf4j.Logger; @@ -44,6 +44,11 @@ public class LoggerUtils { */ public static final String TASK_LOGGER_THREAD_NAME = "TaskLogInfo"; + /** + * Task Logger Thread's name + */ + public static final String TASK_APPID_LOG_FORMAT = "[taskAppId="; + /** * build job id * @@ -58,7 +63,7 @@ public class LoggerUtils { int processInstId, int taskId){ // - [taskAppId=TASK_79_4084_15210] - return String.format(" - [taskAppId=%s-%s-%s-%s]",affix, + return String.format(" - %s%s-%s-%s-%s]",TASK_APPID_LOG_FORMAT,affix, processDefId, processInstId, taskId); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtil.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java similarity index 93% rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtil.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java index 948e92cb24..eab6c4f124 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtil.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.server.utils; +package org.apache.dolphinscheduler.common.utils; import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.Constants; @@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.common.Constants; /** * sensitive log Util */ -public class SensitiveLogUtil { +public class SensitiveLogUtils { /** * @param dataSourcePwd data source password diff --git a/dolphinscheduler-common/src/main/resources/logback.xml b/dolphinscheduler-common/src/main/resources/logback.xml new file mode 100644 index 0000000000..7f634da975 --- /dev/null +++ b/dolphinscheduler-common/src/main/resources/logback.xml @@ -0,0 +1,169 @@ + + + + + + + + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + + + + + + ${log.base}/dolphinscheduler-master.log + + + ${log.base}/dolphinscheduler-master.%d{yyyy-MM-dd_HH}.%i.log + 168 + 200MB + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + + + + + + + + + INFO + + + + taskAppId + ${log.base} + + + + ${log.base}/${taskAppId}.log + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n + + UTF-8 + + true + + + + + ${log.base}/dolphinscheduler-worker.log + + INFO + + + + ${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log + 168 + 200MB + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n + + UTF-8 + + + + + + + + ${log.base}/dolphinscheduler-alert.log + + ${log.base}/dolphinscheduler-alert.%d{yyyy-MM-dd_HH}.%i.log + 20 + 64MB + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + + + + + + + ${log.base}/dolphinscheduler-api-server.log + + INFO + + + ${log.base}/dolphinscheduler-api-server.%d{yyyy-MM-dd_HH}.%i.log + 168 + 64MB + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/log/SensitiveDataConverterTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java similarity index 93% rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/log/SensitiveDataConverterTest.java rename to dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java index fb564a22fb..5d120d22ca 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/log/SensitiveDataConverterTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.server.worker.log; +package org.apache.dolphinscheduler.common.log; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.server.utils.SensitiveLogUtil; +import org.apache.dolphinscheduler.common.utils.SensitiveLogUtils; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -79,7 +79,7 @@ public class SensitiveDataConverterTest { String password = matcher.group(); - String maskPassword = SensitiveLogUtil.maskDataSourcePwd(password); + String maskPassword = SensitiveLogUtils.maskDataSourcePwd(password); matcher.appendReplacement(sb, maskPassword); } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtilTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtilsTest.java similarity index 79% rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtilTest.java rename to dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtilsTest.java index 2e5bfcf3e5..a9a22c842d 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/SensitiveLogUtilTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtilsTest.java @@ -14,15 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.server.utils; +package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.SensitiveLogUtils; import org.junit.Assert; import org.junit.Test; -public class SensitiveLogUtilTest { +public class SensitiveLogUtilsTest { @Test public void testMaskDataSourcePwd() { @@ -30,8 +31,8 @@ public class SensitiveLogUtilTest { String password = "123456"; String emptyPassword = ""; - Assert.assertEquals(Constants.PASSWORD_DEFAULT, SensitiveLogUtil.maskDataSourcePwd(password)); - Assert.assertEquals("", SensitiveLogUtil.maskDataSourcePwd(emptyPassword)); + Assert.assertEquals(Constants.PASSWORD_DEFAULT, SensitiveLogUtils.maskDataSourcePwd(password)); + Assert.assertEquals("", SensitiveLogUtils.maskDataSourcePwd(emptyPassword)); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java index 0b621a9bb0..fd0a08cd8e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.server.utils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.CommonUtils; +import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java index 5f66c3477d..f179d6344a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java @@ -35,8 +35,8 @@ import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.permission.PermissionCheck; -import org.apache.dolphinscheduler.server.utils.LoggerUtils; -import org.apache.dolphinscheduler.server.worker.log.TaskLogDiscriminator; +import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.apache.dolphinscheduler.common.log.TaskLogDiscriminator; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskManager; import org.apache.dolphinscheduler.server.worker.task.TaskProps; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index 8774186bc5..0c62f0648a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.server.utils.LoggerUtils; +import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.server.utils.ProcessUtils; import org.slf4j.Logger; diff --git a/dolphinscheduler-server/src/main/resources/master_logback.xml b/dolphinscheduler-server/src/main/resources/master_logback.xml deleted file mode 100644 index 54c3cf5781..0000000000 --- a/dolphinscheduler-server/src/main/resources/master_logback.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - - - - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - - - - ${log.base}/dolphinscheduler-master.log - - INFO - - - ${log.base}/dolphinscheduler-master.%d{yyyy-MM-dd_HH}.%i.log - 168 - 200MB - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - - - - - - \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/resources/worker_logback.xml b/dolphinscheduler-server/src/main/resources/worker_logback.xml deleted file mode 100644 index 7ba0c9b8ab..0000000000 --- a/dolphinscheduler-server/src/main/resources/worker_logback.xml +++ /dev/null @@ -1,81 +0,0 @@ - - - - - - - - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - - - - INFO - - - - taskAppId - ${log.base} - - - - ${log.base}/${taskAppId}.log - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - true - - - - - - ${log.base}/dolphinscheduler-worker.log - - INFO - - - - ${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log - 168 - 200MB - - 　　　　　 - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - 　　 - - - - - - - - \ No newline at end of file diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 1117fe0015..04c844827f 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.server.utils.LoggerUtils; +import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskManager; import org.apache.dolphinscheduler.server.worker.task.TaskProps; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java index 7cf4b874d1..7da3f710b6 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java @@ -24,7 +24,7 @@ import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.server.utils.LoggerUtils; +import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskManager; import org.apache.dolphinscheduler.server.worker.task.TaskProps; diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index b3310c384f..d4db103fe1 100644 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -57,16 +57,16 @@ pid=$DOLPHINSCHEDULER_LOG_DIR/dolphinscheduler-$command.pid cd $DOLPHINSCHEDULER_HOME if [ "$command" = "api-server" ]; then - LOG_FILE="-Dlogging.config=classpath:apiserver_logback.xml -Dspring.profiles.active=api" + LOG_FILE="-Dserver=api-server -Dspring.profiles.active=api" CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer elif [ "$command" = "master-server" ]; then - LOG_FILE="-Dlogging.config=classpath:master_logback.xml -Ddruid.mysql.usePingMethod=false" + LOG_FILE="-Dserver=master-server -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.master.MasterServer elif [ "$command" = "worker-server" ]; then - LOG_FILE="-Dlogging.config=classpath:worker_logback.xml -Ddruid.mysql.usePingMethod=false" + LOG_FILE="-Dserver=worker-server -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer elif [ "$command" = "alert-server" ]; then - LOG_FILE="-Dlogback.configurationFile=conf/alert_logback.xml" + LOG_FILE="-Dserver=alert-server" CLASS=org.apache.dolphinscheduler.alert.AlertServer elif [ "$command" = "logger-server" ]; then CLASS=org.apache.dolphinscheduler.server.rpc.LoggerServer @@ -93,8 +93,8 @@ case $startStop in exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS" - echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 < /dev/null &" - nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 < /dev/null & + echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 > /dev/null &" + nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 > /dev/null & echo $! > $pid ;; From 8b5c99ef21704cbeaedc95d651ec0a56384260ae Mon Sep 17 00:00:00 2001 From: dailidong Date: Wed, 5 Feb 2020 21:07:07 +0800 Subject: [PATCH 13/43] remove tomcat runtime jar --- dolphinscheduler-api/pom.xml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 35a6f0616a..4dd2f31f8e 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -27,10 +27,6 @@ ${project.artifactId} jar - - 5.5.23 - - org.apache.dolphinscheduler @@ -182,12 +178,6 @@ hadoop-aws - - tomcat - jasper-runtime - ${jasper-runtime.version} - - javax.servlet servlet-api From 7729e0b6b0d0f5aba7542de6533ffeac5e0a61c3 Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 6 Feb 2020 00:22:21 +0800 Subject: [PATCH 14/43] add UT --- .../common/log/MasterLogFilterTest.java | 103 +++++++++++++ .../log/SensitiveDataConverterTest.java | 115 +++++++++++++-- .../common/log/TaskLogDiscriminatorTest.java | 139 ++++++++++++++++++ .../common/log/TaskLogFilterTest.java | 104 +++++++++++++ .../common/log/WorkerLogFilterTest.java | 103 +++++++++++++ .../common/utils/LoggerUtilsTest.java | 27 ++++ .../common/utils/SensitiveLogUtilsTest.java | 1 - pom.xml | 1 + 8 files changed, 578 insertions(+), 15 deletions(-) create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/MasterLogFilterTest.java create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/WorkerLogFilterTest.java create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LoggerUtilsTest.java diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/MasterLogFilterTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/MasterLogFilterTest.java new file mode 100644 index 0000000000..36c08f405b --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/MasterLogFilterTest.java @@ -0,0 +1,103 @@ +package org.apache.dolphinscheduler.common.log; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.IThrowableProxy; +import ch.qos.logback.classic.spi.LoggerContextVO; +import ch.qos.logback.core.spi.FilterReply; +import org.apache.dolphinscheduler.common.Constants; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Marker; +import java.util.Map; + + +public class MasterLogFilterTest { + + @Test + public void decide() { + MasterLogFilter masterLogFilter = new MasterLogFilter(); + + + FilterReply filterReply = masterLogFilter.decide(new ILoggingEvent() { + @Override + public String getThreadName() { + return Constants.THREAD_NAME_MASTER_SERVER; + } + + @Override + public Level getLevel() { + return Level.INFO; + } + + @Override + public String getMessage() { + return "master insert into queue success, task : shell2"; +// return "consume tasks: [2_177_2_704_-1],there still have 0 tasks need to be executed"; + } + + @Override + public Object[] getArgumentArray() { + return new Object[0]; + } + + @Override + public String getFormattedMessage() { + return "master insert into queue success, task : shell2"; + } + + @Override + public String getLoggerName() { + return null; + } + + @Override + public LoggerContextVO getLoggerContextVO() { + return null; + } + + @Override + public IThrowableProxy getThrowableProxy() { + return null; + } + + @Override + public StackTraceElement[] getCallerData() { + return new StackTraceElement[0]; + } + + @Override + public boolean hasCallerData() { + return false; + } + + @Override + public Marker getMarker() { + return null; + } + + @Override + public Map getMDCPropertyMap() { + return null; + } + + @Override + public Map getMdc() { + return null; + } + + @Override + public long getTimeStamp() { + return 0; + } + + @Override + public void prepareForDeferredProcessing() { + + } + }); + + Assert.assertEquals(FilterReply.ACCEPT, filterReply); + + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java index 5d120d22ca..727ab41002 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java @@ -17,13 +17,19 @@ package org.apache.dolphinscheduler.common.log; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.IThrowableProxy; +import ch.qos.logback.classic.spi.LoggerContextVO; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.SensitiveLogUtils; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.Marker; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -36,26 +42,106 @@ public class SensitiveDataConverterTest { */ private final Pattern pwdPattern = Pattern.compile(Constants.DATASOURCE_PASSWORD_REGEX); + private final String logMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," + + "\"database\":\"carbond\"," + + "\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," + + "\"user\":\"view\"," + + "\"password\":\"view1\"}"; + + private final String maskLogMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," + + "\"database\":\"carbond\"," + + "\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," + + "\"user\":\"view\"," + + "\"password\":\"******\"}"; + @Test + public void convert() { + SensitiveDataConverter sensitiveDataConverter = new SensitiveDataConverter(); + String result = sensitiveDataConverter.convert(new ILoggingEvent() { + @Override + public String getThreadName() { + return null; + } + + @Override + public Level getLevel() { + return Level.INFO; + } + + @Override + public String getMessage() { + return null; + } + + @Override + public Object[] getArgumentArray() { + return new Object[0]; + } + + @Override + public String getFormattedMessage() { + return logMsg; + } + + @Override + public String getLoggerName() { + return null; + } + + @Override + public LoggerContextVO getLoggerContextVO() { + return null; + } + + @Override + public IThrowableProxy getThrowableProxy() { + return null; + } + + @Override + public StackTraceElement[] getCallerData() { + return new StackTraceElement[0]; + } + + @Override + public boolean hasCallerData() { + return false; + } + + @Override + public Marker getMarker() { + return null; + } + + @Override + public Map getMDCPropertyMap() { + return null; + } + + @Override + public Map getMdc() { + return null; + } + + @Override + public long getTimeStamp() { + return 0; + } + + @Override + public void prepareForDeferredProcessing() { + + } + }); + + Assert.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg)); + + } /** * mask sensitive logMsg - sql task datasource password */ @Test public void testPwdLogMsgConverter() { - - String logMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," + - "\"database\":\"carbond\"," + - "\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," + - "\"user\":\"view\"," + - "\"password\":\"view1\"}"; - - String maskLogMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," + - "\"database\":\"carbond\"," + - "\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," + - "\"user\":\"view\"," + - "\"password\":\"******\"}"; - - logger.info("parameter : {}", logMsg); logger.info("parameter : {}", passwordHandler(pwdPattern, logMsg)); @@ -89,4 +175,5 @@ public class SensitiveDataConverterTest { } + } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java new file mode 100644 index 0000000000..3cd477f2b5 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java @@ -0,0 +1,139 @@ +package org.apache.dolphinscheduler.common.log; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.IThrowableProxy; +import ch.qos.logback.classic.spi.LoggerContextVO; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Marker; + +import java.util.Map; + +import static org.junit.Assert.*; + +public class TaskLogDiscriminatorTest { + + /** + * log base + */ + private String logBase = "logs"; + + TaskLogDiscriminator taskLogDiscriminator; + + @Before + public void before(){ + taskLogDiscriminator = new TaskLogDiscriminator(); + taskLogDiscriminator.setLogBase("logs"); + taskLogDiscriminator.setKey("123"); + } + + @Test + public void getDiscriminatingValue() { + String result = taskLogDiscriminator.getDiscriminatingValue(new ILoggingEvent() { + @Override + public String getThreadName() { + return null; + } + + @Override + public Level getLevel() { + return null; + } + + @Override + public String getMessage() { + return null; + } + + @Override + public Object[] getArgumentArray() { + return new Object[0]; + } + + @Override + public String getFormattedMessage() { + return null; + } + + @Override + public String getLoggerName() { + return "[taskAppId=TASK-1-1-1"; + } + + @Override + public LoggerContextVO getLoggerContextVO() { + return null; + } + + @Override + public IThrowableProxy getThrowableProxy() { + return null; + } + + @Override + public StackTraceElement[] getCallerData() { + return new StackTraceElement[0]; + } + + @Override + public boolean hasCallerData() { + return false; + } + + @Override + public Marker getMarker() { + return null; + } + + @Override + public Map getMDCPropertyMap() { + return null; + } + + @Override + public Map getMdc() { + return null; + } + + @Override + public long getTimeStamp() { + return 0; + } + + @Override + public void prepareForDeferredProcessing() { + + } + }); + Assert.assertEquals("1/1/", result); + } + + @Test + public void start() { + taskLogDiscriminator.start(); + Assert.assertEquals(true, taskLogDiscriminator.isStarted()); + } + + @Test + public void getKey() { + Assert.assertEquals("123", taskLogDiscriminator.getKey()); + } + + @Test + public void setKey() { + + taskLogDiscriminator.setKey("123"); + } + + @Test + public void getLogBase() { + Assert.assertEquals("logs", taskLogDiscriminator.getLogBase()); + } + + @Test + public void setLogBase() { + taskLogDiscriminator.setLogBase("logs"); + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java new file mode 100644 index 0000000000..94920128d5 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java @@ -0,0 +1,104 @@ +package org.apache.dolphinscheduler.common.log; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.IThrowableProxy; +import ch.qos.logback.classic.spi.LoggerContextVO; +import ch.qos.logback.core.spi.FilterReply; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Marker; + +import java.util.Map; + + +public class TaskLogFilterTest { + + @Test + public void decide() { + TaskLogFilter taskLogFilter = new TaskLogFilter(); + + + FilterReply filterReply = taskLogFilter.decide(new ILoggingEvent() { + @Override + public String getThreadName() { + return LoggerUtils.TASK_LOGGER_THREAD_NAME; + } + + @Override + public Level getLevel() { + return Level.INFO; + } + + @Override + public String getMessage() { + return "raw script : echo 222"; + } + + @Override + public Object[] getArgumentArray() { + return new Object[0]; + } + + @Override + public String getFormattedMessage() { + return "raw script : echo 222"; + } + + @Override + public String getLoggerName() { + return null; + } + + @Override + public LoggerContextVO getLoggerContextVO() { + return null; + } + + @Override + public IThrowableProxy getThrowableProxy() { + return null; + } + + @Override + public StackTraceElement[] getCallerData() { + return new StackTraceElement[0]; + } + + @Override + public boolean hasCallerData() { + return false; + } + + @Override + public Marker getMarker() { + return null; + } + + @Override + public Map getMDCPropertyMap() { + return null; + } + + @Override + public Map getMdc() { + return null; + } + + @Override + public long getTimeStamp() { + return 0; + } + + @Override + public void prepareForDeferredProcessing() { + + } + }); + + Assert.assertEquals(FilterReply.ACCEPT, filterReply); + + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/WorkerLogFilterTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/WorkerLogFilterTest.java new file mode 100644 index 0000000000..ae5a79b626 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/WorkerLogFilterTest.java @@ -0,0 +1,103 @@ +package org.apache.dolphinscheduler.common.log; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.IThrowableProxy; +import ch.qos.logback.classic.spi.LoggerContextVO; +import ch.qos.logback.core.spi.FilterReply; +import org.apache.dolphinscheduler.common.Constants; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Marker; + +import java.util.Map; + + +public class WorkerLogFilterTest { + + @Test + public void decide() { + WorkerLogFilter workerLogFilter = new WorkerLogFilter(); + + + FilterReply filterReply = workerLogFilter.decide(new ILoggingEvent() { + @Override + public String getThreadName() { + return Constants.THREAD_NAME_WORKER_SERVER; + } + + @Override + public Level getLevel() { + return Level.INFO; + } + + @Override + public String getMessage() { + return "consume tasks: [2_177_2_704_-1],there still have 0 tasks need to be executed"; + } + + @Override + public Object[] getArgumentArray() { + return new Object[0]; + } + + @Override + public String getFormattedMessage() { + return "consume tasks: [2_177_2_704_-1],there still have 0 tasks need to be executed"; + } + + @Override + public String getLoggerName() { + return null; + } + + @Override + public LoggerContextVO getLoggerContextVO() { + return null; + } + + @Override + public IThrowableProxy getThrowableProxy() { + return null; + } + + @Override + public StackTraceElement[] getCallerData() { + return new StackTraceElement[0]; + } + + @Override + public boolean hasCallerData() { + return false; + } + + @Override + public Marker getMarker() { + return null; + } + + @Override + public Map getMDCPropertyMap() { + return null; + } + + @Override + public Map getMdc() { + return null; + } + + @Override + public long getTimeStamp() { + return 0; + } + + @Override + public void prepareForDeferredProcessing() { + + } + }); + + Assert.assertEquals(FilterReply.ACCEPT, filterReply); + + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LoggerUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LoggerUtilsTest.java new file mode 100644 index 0000000000..ceacaa99b2 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LoggerUtilsTest.java @@ -0,0 +1,27 @@ +package org.apache.dolphinscheduler.common.utils; + +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class LoggerUtilsTest { + private Logger logger = LoggerFactory.getLogger(LoggerUtilsTest.class); + + @Test + public void buildTaskId() { + + String taskId = LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX,79,4084,15210); + + Assert.assertEquals(" - [taskAppId=TASK-79-4084-15210]", taskId); + } + + @Test + public void getAppIds() { + List appIdList = LoggerUtils.getAppIds("Running job: application_1_1",logger); + Assert.assertEquals("application_1_1", appIdList.get(0)); + + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtilsTest.java index a9a22c842d..03880b69cc 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtilsTest.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.SensitiveLogUtils; import org.junit.Assert; import org.junit.Test; diff --git a/pom.xml b/pom.xml index 742c15e636..92c92ac7fa 100644 --- a/pom.xml +++ b/pom.xml @@ -674,6 +674,7 @@ **/common/utils/*.java + **/common/log/*.java **/common/threadutils/*.java **/common/graph/*.java **/common/queue/*.java From 8ab680f4854aa55131bc895c7e3be78002c4d4ec Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 6 Feb 2020 00:26:47 +0800 Subject: [PATCH 15/43] add license --- .../common/log/MasterLogFilterTest.java | 17 ++++++++++++++++- .../common/log/TaskLogDiscriminatorTest.java | 16 ++++++++++++++++ .../common/log/TaskLogFilterTest.java | 16 ++++++++++++++++ .../common/log/WorkerLogFilterTest.java | 16 ++++++++++++++++ .../common/utils/LoggerUtilsTest.java | 16 ++++++++++++++++ 5 files changed, 80 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/MasterLogFilterTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/MasterLogFilterTest.java index 36c08f405b..8cf6cfc2df 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/MasterLogFilterTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/MasterLogFilterTest.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.dolphinscheduler.common.log; import ch.qos.logback.classic.Level; @@ -11,7 +27,6 @@ import org.junit.Test; import org.slf4j.Marker; import java.util.Map; - public class MasterLogFilterTest { @Test diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java index 3cd477f2b5..8745a4f6b4 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.dolphinscheduler.common.log; import ch.qos.logback.classic.Level; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java index 94920128d5..52767074da 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.dolphinscheduler.common.log; import ch.qos.logback.classic.Level; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/WorkerLogFilterTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/WorkerLogFilterTest.java index ae5a79b626..90b154407f 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/WorkerLogFilterTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/WorkerLogFilterTest.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.dolphinscheduler.common.log; import ch.qos.logback.classic.Level; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LoggerUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LoggerUtilsTest.java index ceacaa99b2..5a80e388ba 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LoggerUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LoggerUtilsTest.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.dolphinscheduler.common.utils; import org.junit.Assert; From 03ce394f3fa5110bea9e9b255316fe5fbf28189e Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 6 Feb 2020 18:05:33 +0800 Subject: [PATCH 16/43] remove jasper-runtime jar, not need anymore --- dolphinscheduler-api/pom.xml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index fd56fb8d9f..bac74da0b0 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -27,10 +27,6 @@ ${project.artifactId} jar - - 5.5.23 - - org.apache.dolphinscheduler @@ -182,12 +178,6 @@ hadoop-aws - - tomcat - jasper-runtime - ${jasper-runtime.version} - - javax.servlet servlet-api From 264a8d2257eb30cb77732f311f6e170c40d7b544 Mon Sep 17 00:00:00 2001 From: dailidong Date: Sun, 16 Feb 2020 00:10:30 +0800 Subject: [PATCH 17/43] =?UTF-8?q?sovle=20jar=20conflict=EF=BC=8C=20remove?= =?UTF-8?q?=20servlet-api=202.5=20jar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dolphinscheduler-api/pom.xml | 28 ++++++++++++++++++++++++++-- dolphinscheduler-common/pom.xml | 5 ----- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 6440805af7..111cb336a0 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -146,6 +146,12 @@ org.apache.hadoop hadoop-common + + + javax.servlet + servlet-api + + @@ -156,11 +162,23 @@ org.apache.hadoop hadoop-hdfs + + + servlet-api + javax.servlet + + org.apache.hadoop hadoop-yarn-common + + + servlet-api + javax.servlet + + @@ -168,14 +186,20 @@ hadoop-aws - + org.mortbay.jetty jsp-2.1 + + + jsp-api-2.1 + org.mortbay.jetty + + diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index 2ef61d7f36..e7789f724b 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -246,11 +246,6 @@ - - javax.servlet - javax.servlet-api - - org.apache.hadoop hadoop-hdfs From cd52040a9b41390706690dee45294da4429960cc Mon Sep 17 00:00:00 2001 From: dailidong Date: Sun, 16 Feb 2020 19:37:51 +0800 Subject: [PATCH 18/43] remove servlet 2.5 --- dolphinscheduler-api/pom.xml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 111cb336a0..41971734ec 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -186,18 +186,13 @@ hadoop-aws - - org.mortbay.jetty jsp-2.1 - jsp-api-2.1 org.mortbay.jetty + servlet-api-2.5 From 139805b65bb34514eca6a0d979684c8b10e8b18e Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 5 Mar 2020 00:08:03 +0800 Subject: [PATCH 19/43] add embedded database h2 and update embedded zookeeper --- .../logs/dolphinscheduler-alert.log | 0 .../logs/dolphinscheduler-api-server.log | 0 .../logs/dolphinscheduler-master.log | 0 .../logs/dolphinscheduler-worker.log | 0 dolphinscheduler-dao/pom.xml | 19 +++- .../src/main/resources/application.properties | 4 + .../server/zk/StandaloneZKServerForTest.java | 100 ------------------ .../logs/dolphinscheduler-alert.log | 0 .../logs/dolphinscheduler-api-server.log | 0 .../logs/dolphinscheduler-master.log | 0 .../logs/dolphinscheduler-worker.log | 0 dolphinscheduler-service/pom.xml | 5 + .../service/zk}/ZKServer.java | 87 ++++++++------- .../src/main/resources/quartz.properties | 4 + .../service/zk/ZKServerTest.java | 25 +++++ .../test/java/queue/BaseTaskQueueTest.java | 1 + pom.xml | 14 ++- sql/h2.mv.db | Bin 0 -> 110592 bytes 18 files changed, 105 insertions(+), 154 deletions(-) create mode 100644 dolphinscheduler-dao/logs/dolphinscheduler-alert.log create mode 100644 dolphinscheduler-dao/logs/dolphinscheduler-api-server.log create mode 100644 dolphinscheduler-dao/logs/dolphinscheduler-master.log create mode 100644 dolphinscheduler-dao/logs/dolphinscheduler-worker.log delete mode 100644 dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/zk/StandaloneZKServerForTest.java create mode 100644 dolphinscheduler-service/logs/dolphinscheduler-alert.log create mode 100644 dolphinscheduler-service/logs/dolphinscheduler-api-server.log create mode 100644 dolphinscheduler-service/logs/dolphinscheduler-master.log create mode 100644 dolphinscheduler-service/logs/dolphinscheduler-worker.log rename dolphinscheduler-service/src/{test/java/queue => main/java/org/apache/dolphinscheduler/service/zk}/ZKServer.java (67%) create mode 100644 dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java create mode 100644 sql/h2.mv.db diff --git a/dolphinscheduler-dao/logs/dolphinscheduler-alert.log b/dolphinscheduler-dao/logs/dolphinscheduler-alert.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dolphinscheduler-dao/logs/dolphinscheduler-api-server.log b/dolphinscheduler-dao/logs/dolphinscheduler-api-server.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dolphinscheduler-dao/logs/dolphinscheduler-master.log b/dolphinscheduler-dao/logs/dolphinscheduler-master.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dolphinscheduler-dao/logs/dolphinscheduler-worker.log b/dolphinscheduler-dao/logs/dolphinscheduler-worker.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index 20d19410e2..fd43458f87 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -44,6 +44,12 @@ com.baomidou mybatis-plus-boot-starter ${mybatis-plus.version} + + + org.apache.logging.log4j + log4j-to-slf4j + + org.postgresql @@ -71,6 +77,14 @@ log4j-api org.apache.logging.log4j + + org.springframework.boot + spring-boot-starter-tomcat + + + org.apache.logging.log4j + log4j-to-slf4j + @@ -78,7 +92,10 @@ mysql mysql-connector-java - + + com.h2database + h2 + com.alibaba druid diff --git a/dolphinscheduler-dao/src/main/resources/application.properties b/dolphinscheduler-dao/src/main/resources/application.properties index 34bb9f916b..06b0ee94d5 100644 --- a/dolphinscheduler-dao/src/main/resources/application.properties +++ b/dolphinscheduler-dao/src/main/resources/application.properties @@ -23,6 +23,10 @@ spring.datasource.url=jdbc:postgresql://localhost:5432/dolphinscheduler # mysql #spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 +# h2 +#spring.datasource.driver-class-name=org.h2.Driver +#spring.datasource.url=jdbc:h2:file:../sql/h2;AUTO_SERVER=TRUE + spring.datasource.username=test spring.datasource.password=test diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/zk/StandaloneZKServerForTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/zk/StandaloneZKServerForTest.java deleted file mode 100644 index 679862f102..0000000000 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/zk/StandaloneZKServerForTest.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.server.zk; - -import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors; -import org.apache.zookeeper.server.ServerConfig; -import org.apache.zookeeper.server.ZooKeeperServerMain; -import org.apache.zookeeper.server.quorum.QuorumPeerConfig; -import org.junit.Before; -import org.junit.Ignore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.util.Properties; - - -/** - * just for test - */ -@Ignore -public class StandaloneZKServerForTest { - - private static final Logger logger = LoggerFactory.getLogger(StandaloneZKServerForTest.class); - - private static volatile ZooKeeperServerMain zkServer = null; - - - @Before - public void before() { - logger.info("standalone zookeeper server for test service start "); - - ThreadPoolExecutors.getInstance().execute(new Runnable() { - @Override - public void run() { - - //delete zk data dir ? - File zkFile = new File(System.getProperty("java.io.tmpdir"), "zookeeper"); - - startStandaloneServer("2000", zkFile.getAbsolutePath(), "2181", "10", "5"); - } - }); - - } - - - /** - * start zk server - * @param tickTime zookeeper ticktime - * @param dataDir zookeeper data dir - * @param clientPort zookeeper client port - * @param initLimit zookeeper init limit - * @param syncLimit zookeeper sync limit - */ - private void startStandaloneServer(String tickTime, String dataDir, String clientPort, String initLimit, String syncLimit) { - Properties props = new Properties(); - props.setProperty("tickTime", tickTime); - props.setProperty("dataDir", dataDir); - props.setProperty("clientPort", clientPort); - props.setProperty("initLimit", initLimit); - props.setProperty("syncLimit", syncLimit); - - QuorumPeerConfig quorumConfig = new QuorumPeerConfig(); - try { - quorumConfig.parseProperties(props); - - if(zkServer == null ){ - - synchronized (StandaloneZKServerForTest.class){ - if(zkServer == null ){ - zkServer = new ZooKeeperServerMain(); - final ServerConfig config = new ServerConfig(); - config.readFrom(quorumConfig); - zkServer.runFromConfig(config); - } - } - - } - - } catch (Exception e) { - logger.error("start standalone server fail！", e); - } - } - - -} \ No newline at end of file diff --git a/dolphinscheduler-service/logs/dolphinscheduler-alert.log b/dolphinscheduler-service/logs/dolphinscheduler-alert.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dolphinscheduler-service/logs/dolphinscheduler-api-server.log b/dolphinscheduler-service/logs/dolphinscheduler-api-server.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dolphinscheduler-service/logs/dolphinscheduler-master.log b/dolphinscheduler-service/logs/dolphinscheduler-master.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dolphinscheduler-service/logs/dolphinscheduler-worker.log b/dolphinscheduler-service/logs/dolphinscheduler-worker.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml index 7d775d5497..03bb94dc29 100644 --- a/dolphinscheduler-service/pom.xml +++ b/dolphinscheduler-service/pom.xml @@ -48,6 +48,11 @@ + + org.apache.logging.log4j + log4j-core + + org.quartz-scheduler quartz-jobs diff --git a/dolphinscheduler-service/src/test/java/queue/ZKServer.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java similarity index 67% rename from dolphinscheduler-service/src/test/java/queue/ZKServer.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java index 3d503d5691..96331405d4 100644 --- a/dolphinscheduler-service/src/test/java/queue/ZKServer.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java @@ -14,11 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package queue; +package org.apache.dolphinscheduler.service.zk; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.zookeeper.server.ZooKeeperServer; import org.apache.zookeeper.server.ZooKeeperServerMain; import org.apache.zookeeper.server.quorum.QuorumPeerConfig; import org.slf4j.Logger; @@ -26,27 +24,45 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; /** - * just for test + * just speed experience version + * embedded zookeeper service */ public class ZKServer { - private static final Logger logger = LoggerFactory.getLogger(ZKServer.class); private static volatile PublicZooKeeperServerMain zkServer = null; public static final int DEFAULT_ZK_TEST_PORT = 2181; - public static final String DEFAULT_ZK_STR = "localhost:" + DEFAULT_ZK_TEST_PORT; - private static String dataDir = null; private static final AtomicBoolean isStarted = new AtomicBoolean(false); + public static void main(String[] args) { + if(!isStarted()){ + ZKServer.start(); + + /** + * register hooks, which are called before the process exits + */ + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + @Override + public void run() { + stop(); + } + })); + }else{ + logger.info("zk server aleady started"); + } + } + + /** + * start service + */ public static void start() { try { startLocalZkServer(DEFAULT_ZK_TEST_PORT); @@ -79,7 +95,8 @@ public class ZKServer { * @param port The port to listen on */ public static void startLocalZkServer(final int port) { - startLocalZkServer(port, org.apache.commons.io.FileUtils.getTempDirectoryPath() + File.separator + "test-" + System.currentTimeMillis()); + + startLocalZkServer(port, System.getProperty("user.dir") +"/zookeeper_data", ZooKeeperServer.DEFAULT_TICK_TIME,"20"); } /** @@ -87,48 +104,28 @@ public class ZKServer { * * @param port The port to listen on * @param dataDirPath The path for the Zk data directory + * @param tickTime zk tick time + * @param maxClientCnxns zk max client connections */ - private static synchronized void startLocalZkServer(final int port, final String dataDirPath) { + private static synchronized void startLocalZkServer(final int port, final String dataDirPath,final int tickTime,String maxClientCnxns) { if (zkServer != null) { throw new RuntimeException("Zookeeper server is already started!"); } - try { - zkServer = new PublicZooKeeperServerMain(); - logger.info("Zookeeper data path : {} ", dataDirPath); - dataDir = dataDirPath; - final String[] args = new String[]{Integer.toString(port), dataDirPath}; - Thread init = new Thread(new Runnable() { - @Override - public void run() { - try { - zkServer.initializeAndRun(args); - } catch (QuorumPeerConfig.ConfigException e) { - logger.warn("Caught exception while starting ZK", e); - } catch (IOException e) { - logger.warn("Caught exception while starting ZK", e); - } - } - }, "init-zk-thread"); - init.start(); - } catch (Exception e) { - logger.warn("Caught exception while starting ZK", e); - throw new RuntimeException(e); - } - - CuratorFramework zkClient = CuratorFrameworkFactory.builder() - .connectString(DEFAULT_ZK_STR) - .retryPolicy(new ExponentialBackoffRetry(10,100)) - .sessionTimeoutMs(1000 * 30) - .connectionTimeoutMs(1000 * 30) - .build(); + zkServer = new PublicZooKeeperServerMain(); + logger.info("Zookeeper data path : {} ", dataDirPath); + dataDir = dataDirPath; + final String[] args = new String[]{Integer.toString(port), dataDirPath, Integer.toString(tickTime), maxClientCnxns}; try { - zkClient.blockUntilConnected(10, TimeUnit.SECONDS); - zkClient.close(); - } catch (InterruptedException ignore) { + logger.info("Zookeeper server started "); + isStarted.compareAndSet(false, true); + + zkServer.initializeAndRun(args); + } catch (QuorumPeerConfig.ConfigException e) { + logger.warn("Caught exception while starting ZK", e); + } catch (IOException e) { + logger.warn("Caught exception while starting ZK", e); } - isStarted.compareAndSet(false, true); - logger.info("zk server started"); } /** diff --git a/dolphinscheduler-service/src/main/resources/quartz.properties b/dolphinscheduler-service/src/main/resources/quartz.properties index 9c8930b647..b01be394c6 100644 --- a/dolphinscheduler-service/src/main/resources/quartz.properties +++ b/dolphinscheduler-service/src/main/resources/quartz.properties @@ -26,6 +26,10 @@ org.quartz.dataSource.myDs.URL = jdbc:postgresql://localhost:5432/dolphinschedul # mysql #org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver #org.quartz.dataSource.myDs.URL = jdbc:mysql://localhost:3306/dolphinscheduler?characterEncoding=utf8 +#h2 +#org.quartz.dataSource.myDs.driver=org.h2.Driver +#org.quartz.dataSource.myDs.URL=jdbc:h2:file:/Users/stone/work/myworkspace/incubator-dolphinscheduler/h2;AUTO_SERVER=TRUE + org.quartz.dataSource.myDs.user = test org.quartz.dataSource.myDs.password = test diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java new file mode 100644 index 0000000000..0a902b1e3c --- /dev/null +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java @@ -0,0 +1,25 @@ +package org.apache.dolphinscheduler.service.zk; + +import org.junit.Ignore; +import org.junit.Test; + +import static org.junit.Assert.*; + +@Ignore +public class ZKServerTest { + + @Test + public void start() { + //ZKServer is a process, can't unit test + } + + @Test + public void isStarted() { + + } + + @Test + public void stop() { + ZKServer.stop(); + } +} \ No newline at end of file diff --git a/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java b/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java index a0cc457e22..97ab9969a3 100644 --- a/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java +++ b/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java @@ -18,6 +18,7 @@ package queue; import org.apache.dolphinscheduler.service.queue.ITaskQueue; import org.apache.dolphinscheduler.service.queue.TaskQueueFactory; +import org.apache.dolphinscheduler.service.zk.ZKServer; import org.junit.*; /** diff --git a/pom.xml b/pom.xml index 8a1e9b98f2..57063a86c2 100644 --- a/pom.xml +++ b/pom.xml @@ -72,7 +72,7 @@ 5.0.5 1.2.61 1.1.14 - 1.3.163 + 1.4.200 1.6 1.1.1 4.4.1 @@ -200,13 +200,6 @@ ${spring.version} test - - com.h2database - h2 - ${h2.version} - test - - org.apache.dolphinscheduler dolphinscheduler-server @@ -349,6 +342,11 @@ ${mysql.connector.version} test + + com.h2database + h2 + ${h2.version} + org.slf4j slf4j-api diff --git a/sql/h2.mv.db b/sql/h2.mv.db new file mode 100644 index 0000000000000000000000000000000000000000..42ea540bd37b54496241f1669d3455f3b9806cf4 GIT binary patch literal 110592 zcmeIbdz2hkeJ9>snrr-$(ZmKDzvyXO9?2e!x*q)+4E8*Fq?YETo*7xPWZFI5)zaY6 zj64rpLMX^@gTcl;Y-|+A7#r9uXE~gcO*Y|`B{^9qyGb}Wu+V7&#_TVTBf)>~k`1=d?&y#>}=V7&#_TVTBf)?45g)&kBX zza^Fd7<~&io$5bQJ35dr;BjmqSI_h<+Q(M0MBr%cki7zz!Sd45>VQ=*Vx0i~nVHO7 z-|GAkdmvXZjYdB2T?=6K9i3ksURqmR9mu54^*_3-;E_w~iSv?)u0O&1o^K|)UOtuR zUHd3|ZrQV!J-0shb@sgRiVAa)$NH@z|5~lhE!h2f;Sv_Mw%Ax2TRPNlBobYBy!jo8%M%x}2JOch2*Z6fS$;ap zT+i@__%isH$sfFYJJ>GS1KSEzI9zkAv;?92SAlk15Xdx)&4mIzS=NU`sd@3IKzVZ8TLl=zVL8Yb$nR zaCNV{M8xo7#PDF`1(!0hBxOLa2~bMXymV!}WKpR4Fj}YwVl*d+av&Q(qv@p);CFER zCd&PRrK9%Z%yMmUrB+|X!k3k6*slU1STw^H%g8F5Jd~qJ&jp7{bOncsw8h|1p4O0y zbOOsYjL5WBa40XAE&~dxktjDG94gWlfkZ77q$$GJ+Q6xy@{aPvOlo2h|DPQj>${Nl zQ|Z#gNOh(>UP_B_@$>ND7(xas)rmnGHoSANGB`X_s-*S~R`!=Cc5E>+nXMx3czJrX zTq#w{6Vo%5bZUBhaBK`EiO^k>L)DSNnL!k=f2K5e;nwc1-c--Vdxk5e!I@HOs!|?D zmZ{yP{i*cS?9fU)l8=*Cp=+$Dl@lNXDa0#J4%)5^p^DW@Xpc*r?W4W z76C2aJ1Ud2Q)v{wCAE3uOUc18TA7)_p|MhE)~Ugfk<{=ct<@lHIhQ>5M@pTi3LR3D zDo>1*_IWZ^>)qY8HI>@Cg+S2yDk0EHcX#yxfuE0qz~Rc|gyaFl@->e?MsI1~RHZaM zU7nl>(s6I(n?Gm+R;a zvtcx?%-p%Ppt+68m+Z@J0M!1U^BPsm)?*F;eNI|ej?GuJ0&Tvc*L2EyOs8{9KWzT; zrJvhCsQrIyA~Ez%_9P3Z+4F)6zQvv!f5e}c&hzKx-~aX%^xt;!7NV0fsgdJ}#AW~H zyoWYik$FxX$+{!qpS-r?JG~uOsCS*81+qKsFmui)fz_8 zwzEYuZ{>>lMsI58;PlQ^ZzfYKwMVF(@P z4S@ekBnJFl!|63YLUY<8pmPf{LtMtiOb}nlCQAEes?5j@u$3`{+#3;Co~)E-0Cse- zC77(DGnIDM^b7HwGE>0~%B({uzl$E^RAv;R|R$3Xzk zL4boGm8Ld`<3bPyi5MxlLDcQQZ42sR>)HgH;JvYJ0u%1K)_DY~aaBK(&oYp#mezs% zb}WRhZkPz4k1`Qj%7{0x0KE~0f}|sau1JFcO@Ze7HJA+Fm)3jK+``ZpEFw|fS=h`9 z#jmDjXY~4}oVqUU$RRrv=#cd_M8`>B--VC-Mj;$kxMuT*OEC3brv0R35dZ|vj94!8LlroS> zrql7*3yLDxD{~VBL`pp>2Li7EJD-ok&Z$9o^e{KfCtBeVnK}!`y1CF<*mlBy;nqar zlC|@{3r8hlWuhxk&|H^jGqM&SJ!2m=?puk5-Bnt8r1XlqB|257u3w3!o7&r_5 z!+luUA2MTB%(l2KWQAV!F&SO5FgP(>>PX>;M)53~aCD9svhZnludB!(1Kd>ILjDMy z_=Vaz$^=zy{bM2?e8a(xUlAVe$Ym<`YKY&V*eW+ry{rWYE0i=3sL)V9DU0$GdotQR5|OrP* zO%Crq3yn*YhiWy;GaE2tFF7qdixd=gMwD*g)&JN#Zyu&+=QYYmI^4Mwt zOOvpWrSpnFH+M*R(2qjp0@$cR zM`uv!ju0$coHsC9JfQULDE&l?CP(RFX>Ug)sLO$aNyvbNC*pvRwZN-FMk^Wo76~YbZ{1{*QgG~^7AddtJTIzwN|g&D=XF2rMv9K>U^WRV&A=HFV^ke7B>c6 z633h#N@nif3tlMQQbAX5Zpm3&-6GvsSffmLS8$_@{)KKIp9+e}5gaF`Sg#e8C$kHR z;95lphG(?xSu!DhWji(WtWQJ(S~!iyrFa;CDR6&?DbSiMo<)G-Jw$F{MMaPPvvm0X zHeluDHcJONNwCgmVuEDBUS5qx$4q&KuD)x-{pp>Pw^jLqpAZybh29KXo;L%S3KtD_ z@6dHWY@JX@G9HDF6K3Iy_D1T>^onf8CLxXfY?+Wn!5ZuPN@XnYL$;o@&ac&B49YzL z=N`2;1GT$iqqZ0bTFz>T*0%$#$Ak9_d5{qcr6{g5kQOOM-vNyNaV&jC3FmrHc#zs? zGYZs0%S&rVqtVK_&Kj zXzhbZch>I$J>`iqUFIC6QDl}=*o8Q*D36)J>D_R;OtEE*QR;3K3%+O3jnd70Scl2i zV>;k4##_p(vO5TtP|wlrc-fno=! zFqV=y3`sl@p|K_IV@V;3>`Vv;sSJ)l27eSsXVbL`)049ma4VM1Zq!z5D@$w3bvs%T zRc%d&m%h!v_*4LbCC35k+*Rbhq8D1k-3cweCq23sAJ^&JcwZfYc5^>gR1HZkarkDivr5334B*e61_BtIOvA z5X8|sQa`LKb-BAfwo;FQaG`{~s@*e+mTi9yb+afC2IutP4gi*x#>8wV#4Ns5SO}6b zA3m9&B@%B=7|j1xCcDUPGDgP91eqjLWDlv3X);4*$zF0B*+=%1+sOfPklaDuNvh;d z>VGRkMDL3ld8sAFl2TCKnUEH!jP8Pr{xpu-AIF?4o$kdvYN--W^B<|LtlGa2z;H?C6yl}8Hfp%+Mkn2fMmD%R z2~qh0I)_b4?<`?lG(?XIXzQ)m-4QPR>p_{`^zlS?0AuaYrKXF8F8}&Y>KKXiRk6=FERgC!*0$m&eJo0qH(4UGQtL#)E&U~GI|{i9;{VL*n&7s zhv#fNR7fn;X&3MqWhy9Rk^oG#HcNHaG~NF1(WQ!U>iK;K7Qzl84YYm8x*l!#nps`13?q}rVDy#GWtL~Le;YjX)cDF&@QM615<)uq!sSV4-xG}cy5 z&koU@F4X;{;=^t17Guk2v31a+?8He77{67A$1&dU^kuFYQB}NDYJb$y5>R+&>G2T_ zaSl2pW{XsyP4~v`J_ZAwNB&y6oyRmfkB8%Q9ug|?R`pXdI_o-+0$s-pDl1Px2+*=`gXr9XHxhgv)0G!Cw<=}fQ?s9?3@`14wlG3mJtd>xN1 zJ-M*K7iWkZZtR25oQ!b2N)E;vWzjM_6BVR@sO(40c84HVa3>P3qDqpHbR>B@Ao^q+ zh~k`+*_f_eEFD;DGhkx5i3KoTnKZ?U|7_ZF0W9Q#+M}Cz8C!UU4 zwijy5%=WnTaguNw_l^W68)5o}2d+0dYS27Ys$ee7J3w{EFy|g3?*v5dh^=V?5fa|a z5;RqS<^xgPb!Zl`bWz7RVlcWBF#1$SFlrr-@UwdS=DE5q$-$BI;KZ}XU^rm)h#ZkZ zC!%AfJEB?&bEh>$5;pf5E54ZQjf_h-{dNCja+$8btdTiVCkzkau-=3 zN5~>sB1g&HWSOjxRkB9zA@`DF)c;!pzT)p-7!92IanRAq0Ew_UAnb`)2y4kB7Q`rF zo#`s1K=tbY*%vzn8S@mJfuaJy*r@94w3HYIL;~#>s-hZz(1F-0s_5Uj#yluq2zi=U z!p8=D9*YYfH+?MVP=Y#x4pIO*O;kI~9~D>~a4dZ26qo`QhU~BCK4xNfmgKBbF0Tyg2e26Y|Q#{h=tmY z^&L}p?*f9p&=G=WM@G*GJZp{S+2RI8zjl<|T|i8HcYDddHB(h`o<@~uSpZrh-R+Ko za1+M^ihm;}b#7z$8h(H}h=;N{0;qhhGpNvI+iZEPSl!pr=!@x?TFh0IZYfa+k-AeE ztb5F~b3a;N1WMi+yDKaO`UPA@``c4uWeHdb+YagjnlFoZi0RmMfQkSyM}e3Z;;aXE z^j%y-PxeERy9Nfw%7gI2wlVcb%Q)^|^<0HG1%e${A(Qj%pcc7CXu5nm)Va84BETzl z?s0lIa5@nir~b{L0%Kxm;d*_Sfy*cdmdp28I7G3YzBz{bVVI(D1t|PhoTXmUeeY}* zZyB8^+gDcVhwaANg1r(wO()G4clm&$p?uy3k0UqbPK)^>1$CCU9s@<5V2m?RG+ z$-_zVNRm97B=1X-_b16?N%Dat`CyVfo+M8s$%m5U$t3x3l6)jdKAI#SOOmIOP0#Zy!_!C zgg8XuMe%*b>;AE3`>tv{pPbsShpnk%H1G~tf-PhSAQe%7GWUW_pj*av)K&!284WE9 zsL&t~vPUgq0(`t2y_`=_65O=}kQ~VoFq?1E3em$1xhW~VF0VKaRY;g~V` zhP&38KTpe4N^RF+OhqtbaB8ZGo40m*s1Dho)k=ly`{zV}cjA1X5H8YHBcb$q>ZC{t z+|1{w2W(j);bk||n(v0z#3c$bZy6iP1odnR@$LZe1a#*EQEognT`Uz!3Fu6PA_Y{| zd!VxZHjaJE*Z9TMRrX?SvCXu(STi_0IRa5{qku$QRj-98tB&jyJ~Q9KEwM8oAq9Z+UI6Kl zI0grG+{A+sbDl*SyHP{+(b2b;>T_Lt}a`a8bw#fSxocoiU}4LML1mprBfY;N6#D9LwNT5qV4XDCMsp#1(MVE$j?cy}l?W5PV%Y`x075behVVhY#!VCy7S z>`CNA#dd)m0IkwEHomsaXKE*Iv~y$Wdf*Z$p~Zhu+E(c0Hj7%APJV!5RWA4Oz#?QK zWW)>xE91BX>w^@mQA(KsBqmtT0IdwJ4^g;&ERF>$cUX8VSY_m0lD#W>DE)kR@jwaV0<@NOdu zix|@D1n8rXp2r)LA?y9L_2UoUi=%KZdea%HD|io!&_mUhU84E`fNv8 z$8JR`WuKr~%gN2QRLka!B$;M=nr17{K5Cn-v^@FRnMiE@2F>R)0k1`5eu2 zPa6QZdD3YaWl2Q{=g^Pj%uG5 zXm;%swPV$VH`b?UK5uW+Sd61l?erqerJXcK~t|XREUvhreLh@%DF6uf?ezxIO1l&CTgRTt;I$4IPvFDQ_yqkx! zua`Gm+O>zgyx~^yby?Rq{X*I+4|mZ_zavv!#wDaGtJO5)LFo7RZtf8Y*$FH=&= zi#Uzsp4visK*uj@i=zv*Lx?ajSs5Q3LnK;(ve*?qwLHJHJimGz;p3%|^6a<>r`1_n zTV*8J$Syr_B8}OHxdl{BwBEc^Oy(pq(l%e7Ve&~aArgK0|o>hYtr znWlG^#wbzTB-%lbJS?M?x9KO>xews{3UmKu$o`>)r8y}GTuhnt;q`Ge>Ofh4ck$nn=KdydHTa&IQ9O1@@Oy8n$LdjDy+@i5G&#NNRUWO zJp1s~T0%Dx-C9ekDYiCIT>B6ZUXgc6aBnzrn|fyA8ZC$bB`ZCELSJ46BaZAl{ZHlv zpw6_iId+NZjzssxJqJc>Fga7JOV~L?57A29TU%b7Up$oBbmxH+z0`#19q1ir$3JeH zX2(DF_VpfKT3N-628K=(``f<(0pA`mOFu_r>X0RNcmA9bY}F{`}v4oKwYKx-Y&ioT~k9TX9#* ztI$ZF{8K>=d+EOT`fzf7fvo$7pFhVNiM@1Ry!kaj)o=anS2$JdrTgL=Vo~*K1L_Y? zf0a|kUb-*tibvL~Dt+Jk&@B&r>&sUsE|M=GtM`}RTA=3CKM1ne3&`5>njq_okN-)E zwHAAUYOS7r#!4piME7|;a}D0f$h}U>jy7dWBT{X`FZ%iWZ}}fT{dRZ4c|rEiJn%0? zmi?tYJE_Ml)zj0n>F2M!^2!^>pNG}C3Es<16Yv#ye0K0;sg8lw+VUYAf8gYzX8a<2 zii>MU=BU$)+1bb7NVD>NX*kL9xsxY1X$S7ul-R^H^3Q+khu0=jLrJQp*?YOxI5NKo z!Ztnd{|wL$vZqI<9Zd2|_bu;FZmY}WqlRg?0}PhaiAi- zlp&EGN}>t6Q)zF#3`4gtbSSWT0k7Zq$rrC>!1#;jzjSRxy%6;W|Mbe0PQ6ZRzvOy> zd2hb&qc_vB26%}-X+B~|Tz}t}-i}{2hif|CWlcq#E%*KXI~Y9Qz~MR4(?g?w@7v!; zW5^ej@WxZu-2${|94v{K=W0eXJ6GIhnuY8(tC`JiD>RDDZ8a-znN~BGohukUzZycx>O>nkPocC2D5EndF%k9bjJVB?M3SC1Us?pGz-?PGR*yEtQKyUzc^RS#W{y`+smVt~<}?#t6m zNBByA0<#RP_`i{&_xHaoWbjM>2`^__ys>e|_F0VfR<^IKE-l*IsRwQQk>m8Sar zx6d!u*XC-gOUv6@YO}C(Xoc0m=*KIn@%2w@z>cWKb>3>ExdJ%;9aatS>ZMm8K9p(y z_kE|XVlNLrjh8d4hOyl|`)VK)QH?MBm4_SG2UX)qjdBCj|Nc*f!Z>{j3WNUg;*)qe z(`s1fq8gT{#y|hi;Rb}F|Dit{7eZcUtmc15n5aMUzrV{d&0Y}u&WFA*$zosF!#j(O zseG+rSbAp8Xj=75-m>z=TA^s>Yeu1F7;}YYYU0?jW4GRYcRz0W89jEa=c0dp^p>yx z^@qV+$jX2I(f6)RNbP|9FMH6QV)<)5Wxe0%&#>S2@1bJLe_$O_;*kfxl{k;j{oRoG zCY$@~-FX>%Zn=(sZ@qhpeQ*7{zhuuF@A*1^{`Fs8amD7VQdeAc-jUkTe#2r9bB@rj z=0@7P_RnK3?O4B6Q=9bx3fj^4;QR9y}{;2augFkHkXtIaF%QJX+ z1~1PrKb2%O4^W!><}iV9*Fdvx!CoAg>sy$=#~!a8!(VtCnPS~$5scBA#YAeX^|1yLa8Jzc57N5O5a4$JFtYAnHhKkR%&&6 zK;X(X>{o&CY+rrZ##E$v0H0ZhDP$?b3p(1TDIFzR~^x#mQ761*TN~E=dLq*z< z8i};|$h3vvP@dMz1c&l+&5#<2v}R=5$Qqd;ToC0(*2oO0kw_cDBOYo+*4JvOZz8c_ z>4Wcra^zZwYc_*xHiK(6gKIW}Yc_*xHiK(6gKIW}Yc_*xHiK(6qsjCs*O?~QZzk7o zCf9GK2otKEmum7-Oc_Dj-55 z%gu!}I!}wvPgPu^X+&s9eMM*_vkNT@!&oHuJEs$yxwx5JdQ2`oCYK(QOOMH=hl_^A zrN`pZV{z%Rxb#>e3YQ*>OOM5+$Kujsap|$R%v!uuiP|vi$od@&8dZwjhHMpf{H8`Yf zk+~jSw%*e6O6Gctu<>9~F+IBMmNABn8sln@{`ObJqD#28UV*BYCZtXc@qkQVZGL&3X0?nM) zh}6|DC81$>V;WiItKj`Ib>+)azLXNoyb`A7Qe@q)G;h5Tt&H-e6lW=9TBZ_HA;nUv zU@3%I3Sm}(t2EhyKJhb?4b8GNYcAQ;vbv_@f2O8qH9fECMNOa6^oFL}n%>lmjArP` zqGlMHVQPk@8ClK9X+~Z%3Yvl6Yl(!G`JJyMQVE4_g%Me$_F1L&S*7+_rS@5+_Svkr zd6nuGlmZG$vVziR1;HaBvrLh|aYH*0=N(FgS#$kfeh2uq(r&QVy2;pIdV5j)Xd$}Os!h=kK0`aAUWreWtu&h7<|5Ac-0*@j{fJb~O zROS^b^9q%D0Z*Q;pu{ZXyfsjqU`3?{MWqHsh0LOWl+)d4x}oWorsp)hkSu6=P1C_9 zLo>}JIA&>PRx@*&nb*vMW)?LQoV1hRP)4(K%`!C0)be@A^7QW~p|LrRWah>js2i$Z zuZicJc-F}n~zaUGu<_g}gx&dhwVY*$bqhD}h)^iz8 zx`v^A)e0`1mR+~px?9a6G_lMjJEs@CUyY2DPPc7T-;LSGW*l1JpscxHdNX4*>WImm zmzJQKeK}OiFiy4j5>(4CPPOh&@L`_kGOjKCcCbyj#?6%TE;ueo5aVma9IfMMCWW;WAcTXoLap9>vFICJ3Y zbJBzBIK=ZzezOg`E*#?7LLpbrngQNrz2WgLNU%-a_8Fqm*uxCpF87) zav_W0&Y8>6GuYBph-9WI0^5s*B#Lzq0lPsKO&kHX27JF{|5r1Af&!3ln|Jzs4 zf7{7hh)&A5RP#92Z2p_`?#B(AeD_(NoKG$w7vdtD9XmS#F;!FfYnV6gy z#{Xt3^orQ1h~iXFdURT7vk2e7RFA>o|As3f0dd2qEv%tRGAlnK4Wtxs;z+p;EnRyk3jNY_d9W7T% zRen7HKZRFF;8dlwHzLq;;YDdjsUi}|7Gc|w(*ZTqb1hm&rr`&Ew+ecpB?aG2<)zs# zj3r&-g`vmq-`_&ECsw~)+u^pnsFQkZ)o%=g3=X8%#$ zJ{bE1NVX9}C!|?^I}*JG68%z?M2i&D=~&|IKiFH0hngUW1@vqR#Q|x8dN-M-Bk2}f`-UfHS=Um=O;|I2aEdqx5qHsu z)31HKF}tio*1d!ya(&Tg2XZ4YcQQ8S{MVcXw{fS?y9vvL3vM4mnSCzRdA4e;9{8S?YJ!gr#A+rDP*=% zxNQU6-V+~g{%dv_-h3_EDcbtmp-s&{9@hMq@Sa1k(c5~r18pz1xr4@$#meN=>j`k^ z9C7^>XvVd)t$lY^H{^)kiD<-)h)yUnM<`IIXt!r}enmO%uVa%FxY10|sf-ES5F}7S zMoPm@l8M>zQl&h+MK}9WmaLK!0Vx9`P#eU_LKq}s0O1DFV!5rtt_au!8GCPR88acS ztwsUWxGIi16|=5x$l2%HX?3NHSfWPhjW85spL-^H9a-bR+FygLeP3GdQ6|IC7_8UV z(a3_)>l<=*UD}aC9tu>*zJ}-+DVMd=$Z7$xU_vQRBePkQyC)-H|{>rYmfamdjMgIHw z*zh8-@;>IPb)w9_63mZu76oM+OUnIM`UMZWgpC`C0(RBkRRF<%jWf)mcD;jvv*176 zhn4*yBjzG*eeoM~mGTa_N;*;lDmwzkLqV3SbDah2Mq?%tjl%AU^jVr>$imqZ<#vHg zyElPTw~)UHI`Ip&bCh{JF_8i&;=wl@?D!So;f`FUa<7K?9g3}T12wK6c0S2-&_spq z8VKD}Q5x5mSuA;r*1B%_k)p=*?X?iKAIH(R3^TL1G^fg0f5sl1Gr}~xIca!jX?VAD zA$c&gWgHj`!9E#=fwuw!M`B|jaIn?V9zGDL)uc#B0STw0kl@cU78Ih#S3W8t@xX>t zI8tll2jk#jY;t(_S!i6EJXC*Lo*6C1FC6u?u^O9SFko9PDM7D~OD-9dL)U>rk4JHc zXA+AqAR=O5D}C_IT#5T3djsVP|I1*AA^ZleWiZ zH2a;R*&_!SYsgYBw?;&yag zo*3bvN?SaLji{3URzSwe&?@!xNQsP+ z9b_jpI(5|O#g1!4&FXX7xSm=?fZhPHEA0?~rl)Aa6}QtfVkFUudf(vM?;!CFkI+sg zivB>`c0$>#SlY+1Zg+SuT`zK$<)d+e`(X=29gWc*yQ+bTzD*;9ca&L_+g9zPErNT> z7{$8%We!EP)w7i4XH-S0c7hc@Re5=oFRj8*l}F1zlNOf6yVWgVnEjPD)0Qolp2oXU zx5n-RtFA8HWiLiwpe^pQW3wwM?+wSoke->`UBbj;n>*eyN9is+VN=~RgX2>{`Edk` zB`T3Il4v{7M9oJ4!{(+Kg1MhO%~50K=B?TV_ftNf&tC9 z0V^-JSvtr`qF36+1PPAuUyVk`OnHW`zH7t%>7A3eRr!J+nB0L}pM)t+*HFx0j3SW5 zF2RgW#-q@I*era}-n@HpiNJ+)1!{V164L0;mI)d2<#`iL%#MvY{-NPgnO?xCq{XT~ z{%ds@gK|%%=N`2;1GT$^I`;q~1hs*8(zDiqzaNwQRC~Q0Xg!XFtnky#&_$D;tUSnw zh0>01qi2`VzXKTk<5>EPVwCGa;X$J76ckP=SH{;UP!G||@S@Smxz63B60$}%_`+@; zHxjhU`qp;&oqL4b0))sraN8qf)IUey@?5mE#L2C|$*D-K=CUUiT4>Q80%PbjqyS(C zftMHCtOsSa5LaTy(`q;s_(*NB5segi$#x|F#R2c|mw4`|_TkjkC^E|_tj!)*ln1*D zcLb-zPhgs2{}lA80(CcwQ7rhL#rjPwt{9vt?Z9Bt8_`vHw+&Y4QS$+rwZ`E%Z;>yz zQ0Z`t(Hocd5ABP}~W3lx$R+uYxfC^(NiNlb@6A>C);y#uXqR7sKaFEL22xRa_ zadftO<9IwVY~Xyjm8G@ix*aWvsu zsh_u*xMlKCs#KsMB*=Xu0blD%?HZ{+<7geJAJ&z+916!)>M;;5l(1K|dq&Z+?a!fZ z76p!Df$ZQ80G5`<#B3+TEWTA(2$C`%KAE85`jdpg{BLEli|i(2WSmTpNis$DkP4Y5 zGh~+RCAX1%WIwr`93ThD9ps&)O75iow=zWZzNnFxT4F3I1?8OyX@SbRwp0hnrS zmg=r)y8PdxOBLhP^ZO1ggdM~(Wom_}&W+KrL3m0+>V~=KK{^mplZXM5`lI6{QRs1g zJ#*Iym7U31pRBtU4ID^=SR?K?WqL6}_m9}(_f+GPBjr(S#`Gu;)1wl4`QU#7Lc1^a z;NSAEJz!GX%2d~NGYQdsG^%3=NE}OkDJh+G4@iOjVG0#~m#A>7xK9qFyH(6$su6;^ z&;Yu(r17Ja5X^Q^m4XuYKxcg80#RhhN^hkd%QeLrRR*Ima58%SdP88W|tlhnc?pK_t5x;x+F- zkqr@>nc>=8f=`M;YY)+??|XIWG!IsgVla)hmD96Bbf*iIa}^(MW49PvK8vk`9%Uy^ zV!-&VIy{c?hNmxc&4{YvrBeH&mX?6RJ4=s`Xoz#rA+a%E1=@6P?CxVQ(0SyqrQ3N- zqw{z;PUj(^5^vKmC8M*h11X@$XHZ$WvOTQpkd$CF-lpS_F;QLF0`6dus#~`QnMJ|3 zhxA#lz;#rln~e2Qd(lVzIF8i>QfUttyK8!vU2iiHr;cFt3>HTQ3>=j~p)`irn(~0i z?M9(d`hz!fsI?A4Mj9gi(Nxv;?(XNVkb?1RvpjBve5 z4#pZ~(K0&|6{LWu>_^RZhagsPClaorN|KRu7QsBgc zz_PtiV`jF;t&eLMm!pT#5}0g+=^Gw+$naZq9_CJ!Dws?24p7}O%(;iiI{}e9Vr!Z} zgoHP<1Wgs7`9M^69hyZfUDPp-7>w=&j6T&7jAU=@cF=q|Mhi2QnS2LF(t{Jv9)sb4 z)gy953Z00K8S{ow=9Wmp=3ZmP=cC=|yy>RD?w?Gq(e;-#GDqs9L2S|_hsa?vPu@lD zA`9dQStLv3D7l*~lNGW`*2q2NUUH23e`~;3{2dIVfm1&YI$9YZ5jF>eJrN6GEqTO( z7$vMTU4;~=ejOnDVy7Tuo`N$_Q~($oRh^xd62pKa#g%6zqQ^3NI{S_TZ;hOP{|FI*m zVs>G2wgQv+O#0Pjn@^UnRL>5uE)gw?*j@7Ri|thpgLiVLjfeN1agg|xO&~-oJ-2#X zB}mN9$HuHLhghiXSl=;q_bwpl3mqY7c4YL7z_Zq9o-J-r^lL}S-37$Ncej`PTQgN9 z=V?@lmIa_C(%tST2sd#&p!hdpLI|Vh%RoGo%@IK5bDcp2L-#6M04i4Zbu{{7IwtO_ zO1G3Kgh<_~4Awm(`Ex&7Uj$0t8M`Yi1^NYCM*G`SVr2O+ zQs5kyQ$fk0V{Me2Oc0Adw9L-6>_ z@)k_VIIp(J@YNghd(N0a1zN%H6;=oKBM8NRsE04pGuM!ljNl& z`E-(eCP}?W$AOnWe1i~&D7+}XuXx=*)@6Hk#8xS7vU57@Fq!pm-^HQx=bIUT!a8_ERrYzp!20PzHL=L1o0JT+Y` z6-o){Oobu^RMvZ-vi>%XeaqMQ#ne^yVr{X_w7FO_I6OImL4O+sB;u-iEks#$WUuha z(FmCc87X;gVCW(m5dJVn?2wvQ|gN@q;zqtFRg1t_SNA$a#?7{@X=l?2e^ zZ=)VDgA^#oiiG)nN#Ok55jbbjW1*ZD*BQ!@0w}*f37G$vINlu!&6qHcH(ReVFGTw> zftVVcp2pTmtk{!)j*9I9J3ywaiYB&OV&N0)uyTim$AVQxZi*8~ z++P;$0+Km8AXH9JW$pejZTI-5X#s~Hw66S-CNC&XN4_^HKWmpi|2eWp)HZ zScPfJJw@{@w;@sH>G25$SKCC6PtzRR@d7!r1x`_Lh(iVL_PFBu@ zS7d!ZNwaFFE)?ftXm(Wlyg;*Sr>GsPF1)cmMe})ko5o@sjcTVCX)f)gIU2uInqQ*X zw9_um_&lT=qZxggX4I}z^o|5W+nS=_Gc>n$tW&!D@a%Bmb9VmIT^sm?&)5FmDR#m0 zwO{=s{`~Vl;Lrayl-!Wiup;^kN%E5<61`>que*|1K7GmgT?@&dZMdlGIQiLzTM=;c z{CB!GBKZ54n+_lB z(h}rJjhxm*me(m+Zsu7clN$~{NRpyDKf0fj|IxkSWOS4PaI6Itg-VxCzcxa!Suj^gXx3m zJNgc$o3;4`yP^DgGTrB9u`+|{zVtw|wyP^N)TVOrstPOZD2q6iZ60N2?~+%TkKj zZZ+TtSyc%)D9uS#AF3`z#ml>)Z!$eucUJGGzbI!qS`+ z1TLmb`tW+f51;_q43@+biYW11J@UqXa6}al<$~%J({vh$Z`dnLm~j>hA_2kyqSm6j zPIxsJib_Lmd9)V)g=K!!(MBim?Nc-93@uL;_Owo=N{S&05;Dh$bES20ZDFBL6=oFB zPARm=r&swLxZ@m;4mKODF!8D9QE}JbF26_6!UpV#te)4E9 z(wfiy+pDlPb3?3z=OaNPG4bquS8EB~NOWs0sixT4#47i72ZUGTT@pM7a+`Xle2o@F zfRdFSK%p-$gAqseo&KqL0jM*rY>r)Gx+BqjanFI#8cfdA>JrXlr-x{z?yW5^&MzKH zZMyToiC${L^bYinv*RDPO~dH2mwWqq4==5(;%B{R^y|4of4)D5qx*a9d+bH}GoSQH zHJ0vO#IG{H1q3hf(D|jcmEMy$<{;6NNL=*Y58d+0-#q^3{VlUknUl&f*ez&c-tL0T_q)+~-AcwtlUwnNyIln;G{lm|ns6J$?|tZ&hradYs}mQ=7m(Ha%Wo}E zbLt-iS?mR5?RZU)^~J~kB*j{by+E~APd{fR6MCZiyq>uR?_}g&r)5W*vZWEJHsKfj z{QbB5kDq?KJK?+_`)3~be~T>pOM7-wk6Wszr)Sg8UwP$~H;z9Kt8){)mzyTwEAaU2 z;K@=Q1FN;=LpJ`v$wkfhMfemK*N)6lrx&xckHL{<<@?fblI3$JPj1o<+_5RKiD~4Y z|JDz$O{9jBR86z@a;ei4Lidf-O`w1e#F(P;;h{L=kvZ{xjRs%IaibX#T{?Ot`u z%;-iopY7>Q(Oe$;%nMu!{^gr^Nlz)g$R-Vr$y5)z1{(G6{sM(3cy0GQh1&KcQm;;J zujWhs#Y9q>TQ9@VEestBtX{zDH-7TPYZ);9;`uLK z8&NMr{lP!Ia-~zR)7me&USQsv@B7HjbgThh;!m277!udt_rzy(ZBcY@1rr~3rcw7sq1e6S~L!pMCR0mx3XlUXE#p#rDG8Jq~QPT z-~W(H0eeC6^r@XYM0^ww6l2G!H||O%F!t)&c)gvg8O`imahqutvfHd?HoL9RC^om% zth{Af&0Kb_VDyaZM!r9T|8!cX-+k_>Yfw(YI`x^|l+JjWutw0nbN>xvU2J5P=)Rmc zLz#JfrKBFO{cJ-oZH@o+mjE<<`SG{K%h&!9FNzFoym9;Lk)zxFs${!;%&u=2XY6d( z`G2_Tq3f}iv=K-QFxu07d79}6U+GU^mVp)jH&XQe{ssUcT^a{j>GVTAq@6=W7<>9CCa%R;qwwq^P4P+v!@rA$gaO3)*YCNe? zZh-pV|EW+Ir%yp)&|hAB5-(?34eMM~!xGi_=N~%UfKc>5^rz!O$jgk?{O<@8^+*2q zcR8ln3u52-(C$i&hYn9F%_3MR8SlHTPV`*&ZP(Sno z`Nre_n0QO#!usL0#k=}-Q@7pMrt@lK+*h5IqO&6P41b6(gMXR)!Ku=-{42*FdHyKy zN0C2j{4vKLb^d7Zhs_^N_Aq#P1~1RxDXXU5ZUgXNu2YPC>mQO37WDoKmWc%Zo>fou@^|tg1E9G$J&lz9KY|*%&GKa>fWM_q%%%o4L4|TzX6{ zJtmhPlS_}urH6}##ihsM(qnPyvAFbDA_|uti%XBirN`pZV{z%RxXfC-REw8t@lq{b zs>Ms?d1ZO2Szct1Z3;aWxmr%=zdz?}#4 z#CoQsWHq>@Xf-&bY>~MhUAErR@=E4|sEVR+nE!V6`;V7=Cuq7*~7rx4$YT z59C$xH>e2YQ1HV@N$IbnlG4v*b%^ShDiJF1=dX_mf9wx_`>Tno5=@0C7OS4&HmV~w z%BL>uS;;je(9C&FUtRrD5*mg#rjb>?3f?bMSH3LeODVz3D`9FbMb`aFqud+O$|zq- zah5`+WhyZhQY@tkmO_}N5M~v)N|P<<>(I<(L$fT+noBmdtgh+!pQ-6tP0wq3QPbx% zy`ky0rZ+VsqZxX#s2PT4n3`c}MpiR&nvvIxf@a|NS|Xukes3j_N+@(IjL0gr&nmUg zDz(olwa+TG&t|>Nt5mn36i`r-6_iFR2p$QUWr_suCfb2G?@%htn(OylbG&>GcL895 z{fx;mDnE1TXHogo4fQjtehN%UsZs(BUH!Dw&%E+!sx+D^jiwq|B2*L$F3)L3T{9Y* zVQWTHGc%g0C+9S?rkQokY-naPnM+#9Y*J5}nw8b8oMz=UtDsp$&8j7B&8lk_V1^ff zETd%&EenvrQIi@@E}v5Kzu1-Ss^SuEGtmJ zzm%Yyz@rEf;1OR6m3f8Ayh3GOz>}vdC@~8;Zw(aJT~Vn)QK>;uA+snT<#aciZfLrt z={Zd=Bnz5e({!-O&`dK4j#-+S)y$k`<~6gRnMKV6C+#FSl+i3*vkc8LwR|43JpJW^ z(AXSDGIQe%)MM7K*Ti#9JnQ1wIFE&!{L_?AOMDl_QzSP#9Yij7VNDW0^0loxxZ}`mEt+<38)A*Vk$EH5cqvEC*X2FwAVN zX_yVVQ|H|M;~stLvNlEedQCV`J>GHZ)b)FFso%N9N1D%hsfP>LaeSXUl9ik;#JtKQ zmp1qu${!>WdwzLyC||k!f5wSBGm1QUsh{cN-t=3W!d@VjS}v@ISI#cp4pE|KT5`%Z#$ zPfq103yDLwhOv!)Vq~x)&fgo}i94@{aUv)^3u-@>Vr`*kL5aBI<>^t}6VI-9s^G8? z?ES-Xjf@4`;3OV~n%{tbJEdGPNI-&;0sfS

9+dVANTZwqCRi-f@HUQ_ zNG$Ceo}8V)wdCShD<3f;4!|VNq79D54(pV4FX4zBUo?uj8-cl#u`%aA<~+EKJB8j& zfZpe#py%geECl+GM)&aCgZ~hDh(sVgLcI?N{Cpe)(&PIj51ivcJ+uVC;+Kj)MsI1~ z6izsqW`}gfl#lXwsd_t3OT_67>3}*FGFvI!wgGPMi4QmbH9HJ%z838iZT;=grsf|H zYyL}kJPbZtu+iIkwgYW1wmE~=k;Tg7)awawxTBQbvN%=6wKT0*%j$;gPo0RkZ!%s) zbV894p+KFY-JaR`73H|Uj!jPBL~}u>GA3|CkU$9;DGfVGCT7P=mGbZw-Rw(Qyh@J! d@3Ft#wPUc#9S1<(Y#KSEIj7jaXZvyC{|~3$rtbg% literal 0 HcmV?d00001 From de0ea194ab23487f6c5952557ba83e221dc9d68a Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 5 Mar 2020 00:22:19 +0800 Subject: [PATCH 20/43] delete logs --- dolphinscheduler-common/src/main/resources/logback.xml | 2 +- dolphinscheduler-dao/logs/dolphinscheduler-alert.log | 0 dolphinscheduler-dao/logs/dolphinscheduler-api-server.log | 0 dolphinscheduler-dao/logs/dolphinscheduler-master.log | 0 dolphinscheduler-dao/logs/dolphinscheduler-worker.log | 0 dolphinscheduler-service/logs/dolphinscheduler-alert.log | 0 dolphinscheduler-service/logs/dolphinscheduler-api-server.log | 0 dolphinscheduler-service/logs/dolphinscheduler-master.log | 0 dolphinscheduler-service/logs/dolphinscheduler-worker.log | 0 9 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 dolphinscheduler-dao/logs/dolphinscheduler-alert.log delete mode 100644 dolphinscheduler-dao/logs/dolphinscheduler-api-server.log delete mode 100644 dolphinscheduler-dao/logs/dolphinscheduler-master.log delete mode 100644 dolphinscheduler-dao/logs/dolphinscheduler-worker.log delete mode 100644 dolphinscheduler-service/logs/dolphinscheduler-alert.log delete mode 100644 dolphinscheduler-service/logs/dolphinscheduler-api-server.log delete mode 100644 dolphinscheduler-service/logs/dolphinscheduler-master.log delete mode 100644 dolphinscheduler-service/logs/dolphinscheduler-worker.log diff --git a/dolphinscheduler-common/src/main/resources/logback.xml b/dolphinscheduler-common/src/main/resources/logback.xml index 7f634da975..c5cbb3c850 100644 --- a/dolphinscheduler-common/src/main/resources/logback.xml +++ b/dolphinscheduler-common/src/main/resources/logback.xml @@ -140,7 +140,7 @@ - + diff --git a/dolphinscheduler-dao/logs/dolphinscheduler-alert.log b/dolphinscheduler-dao/logs/dolphinscheduler-alert.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/dolphinscheduler-dao/logs/dolphinscheduler-api-server.log b/dolphinscheduler-dao/logs/dolphinscheduler-api-server.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/dolphinscheduler-dao/logs/dolphinscheduler-master.log b/dolphinscheduler-dao/logs/dolphinscheduler-master.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/dolphinscheduler-dao/logs/dolphinscheduler-worker.log b/dolphinscheduler-dao/logs/dolphinscheduler-worker.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/dolphinscheduler-service/logs/dolphinscheduler-alert.log b/dolphinscheduler-service/logs/dolphinscheduler-alert.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/dolphinscheduler-service/logs/dolphinscheduler-api-server.log b/dolphinscheduler-service/logs/dolphinscheduler-api-server.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/dolphinscheduler-service/logs/dolphinscheduler-master.log b/dolphinscheduler-service/logs/dolphinscheduler-master.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/dolphinscheduler-service/logs/dolphinscheduler-worker.log b/dolphinscheduler-service/logs/dolphinscheduler-worker.log deleted file mode 100644 index e69de29bb2..0000000000 From 2fd5898cc86ad0cc0500b392f26a781aee79d573 Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 5 Mar 2020 00:29:51 +0800 Subject: [PATCH 21/43] add license --- .../service/zk/ZKServerTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java index 0a902b1e3c..48cde32287 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.dolphinscheduler.service.zk; import org.junit.Ignore; From 6bd797f3e5f824e9c43b48f7bc2ce8697473d388 Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 5 Mar 2020 15:49:40 +0800 Subject: [PATCH 22/43] change log level to INFO --- dolphinscheduler-common/src/main/resources/logback.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-common/src/main/resources/logback.xml b/dolphinscheduler-common/src/main/resources/logback.xml index c5cbb3c850..7f634da975 100644 --- a/dolphinscheduler-common/src/main/resources/logback.xml +++ b/dolphinscheduler-common/src/main/resources/logback.xml @@ -140,7 +140,7 @@ - + From 5e043c31bc8433c2f07d3726eaee02b7be9d8f8a Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 16 Apr 2020 18:36:32 +0800 Subject: [PATCH 23/43] simplify and optimize config --- .../common/utils/HadoopUtils.java | 60 ++++++++++--------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 5d4f867295..02f00ce330 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -57,7 +57,8 @@ public class HadoopUtils implements Closeable { private static String hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); public static final String resourceUploadPath = PropertyUtils.getString(RESOURCE_UPLOAD_PATH, "/dolphinscheduler"); - + public static final String rmHaIds = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS); + public static final String appAddress = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); private static final String HADOOP_UTILS_KEY = "HADOOP_UTILS_KEY"; @@ -110,14 +111,15 @@ public class HadoopUtils implements Closeable { try { configuration = new Configuration(); - String resUploadStartupType = PropertyUtils.getString(Constants.RESOURCE_STORAGE_TYPE); - ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); + String resourceStorageType = PropertyUtils.getString(Constants.RESOURCE_STORAGE_TYPE); + ResUploadType resUploadType = ResUploadType.valueOf(resourceStorageType); - if (resUploadType == ResUploadType.HDFS) { - if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE,false)) { + if (resUploadType == ResUploadType.HDFS){ + if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE,false)){ System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); - configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION,"kerberos"); + hdfsUser = ""; UserGroupInformation.setConfiguration(configuration); UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)); @@ -166,21 +168,6 @@ public class HadoopUtils implements Closeable { } - String rmHaIds = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS); - String appAddress = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); - //not use resourcemanager - if (rmHaIds.contains(Constants.YARN_RESOURCEMANAGER_HA_XX)){ - yarnEnabled = false; - } else if (!StringUtils.isEmpty(rmHaIds)) { - //resourcemanager HA enabled - appAddress = getAppAddress(appAddress, rmHaIds); - yarnEnabled = true; - logger.info("appAddress : {}", appAddress); - } else { - //single resourcemanager enabled - yarnEnabled = true; - } - configuration.set(Constants.YARN_APPLICATION_STATUS_ADDRESS, appAddress); } catch (Exception e) { logger.error(e.getMessage(), e); } @@ -200,7 +187,29 @@ public class HadoopUtils implements Closeable { * @return url of application */ public String getApplicationUrl(String applicationId) { - return String.format(configuration.get(Constants.YARN_APPLICATION_STATUS_ADDRESS), applicationId); + /** + * if rmHaIds contains xx, it signs not use resourcemanager + * otherwise: + * if rmHaIds is empty, single resourcemanager enabled + * if rmHaIds not empty: resourcemanager HA enabled + */ + String appUrl = ""; + //not use resourcemanager + if (rmHaIds.contains(Constants.YARN_RESOURCEMANAGER_HA_XX)){ + + yarnEnabled = false; + logger.warn("should not step here"); + } else if (!StringUtils.isEmpty(rmHaIds)) { + //resourcemanager HA enabled + appUrl = getAppAddress(appAddress, rmHaIds); + yarnEnabled = true; + logger.info("application url : {}", appUrl); + } else { + //single resourcemanager enabled + yarnEnabled = true; + } + + return String.format(appUrl, applicationId); } /** @@ -484,13 +493,6 @@ public class HadoopUtils implements Closeable { return String.format("%s/udfs", getHdfsTenantDir(tenantCode)); } - /** - * get absolute path and name for file on hdfs - * - * @param tenantCode tenant code - * @param fileName file name - * @return get absolute path and name for file on hdfs - */ /** * get hdfs file name From 0d0d844efe54d9972546f55d912d0bac551b93f1 Mon Sep 17 00:00:00 2001 From: dailidong Date: Wed, 6 May 2020 17:50:31 +0800 Subject: [PATCH 24/43] add development state --- dolphinscheduler-common/src/main/resources/common.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index db3b241ca9..d066f7b6b7 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -62,5 +62,5 @@ yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s # system env path #dolphinscheduler.env.path=env/dolphinscheduler_env.sh - +development.state=false kerberos.expire.time=7 \ No newline at end of file From 175475e0a415bd5440322ee4c71f2157cbcebf39 Mon Sep 17 00:00:00 2001 From: dailidong Date: Wed, 6 May 2020 18:48:28 +0800 Subject: [PATCH 25/43] simplify common config --- .../src/main/resources/common.properties | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index d066f7b6b7..3852c310b1 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -19,22 +19,22 @@ resource.storage.type=NONE # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended -resource.upload.path=/dolphinscheduler +#resource.upload.path=/dolphinscheduler # user data local directory path, please make sure the directory exists and have read write permissions #data.basedir.path=/tmp/dolphinscheduler # whether kerberos starts -hadoop.security.authentication.startup.state=false +#hadoop.security.authentication.startup.state=false # java.security.krb5.conf path -java.security.krb5.conf.path=/opt/krb5.conf +#java.security.krb5.conf.path=/opt/krb5.conf # login user from keytab username -login.user.keytab.username=hdfs-mycluster@ESZ.COM +#login.user.keytab.username=hdfs-mycluster@ESZ.COM # loginUserFromKeytab path -login.user.keytab.path=/opt/hdfs.headless.keytab +#login.user.keytab.path=/opt/hdfs.headless.keytab #resource.view.suffixs #resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties @@ -46,18 +46,18 @@ hdfs.root.user=hdfs fs.defaultFS=hdfs://mycluster:8020 # if resource.storage.type=S3，s3 endpoint -fs.s3a.endpoint=http://192.168.199.91:9010 +#fs.s3a.endpoint=http://192.168.199.91:9010 # if resource.storage.type=S3，s3 access key -fs.s3a.access.key=A3DXS30FO22544RE +#fs.s3a.access.key=A3DXS30FO22544RE # if resource.storage.type=S3，s3 secret key -fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK +#fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK # if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx -# if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ark1 to actual resourcemanager hostname. +# If resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ark1 to actual resourcemanager hostname. yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s # system env path From edab6597f696e9e6ea596f96ec68d5a807fe3eb4 Mon Sep 17 00:00:00 2001 From: dailidong Date: Fri, 8 May 2020 19:05:37 +0800 Subject: [PATCH 26/43] add worker group config --- .../src/main/resources/config/install_config.conf | 12 ++++++------ .../src/main/resources/worker.properties | 4 ++-- script/scp-hosts.sh | 11 +++++++++++ script/start-all.sh | 3 +-- script/stop-all.sh | 3 +-- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf index fab6b32d51..f6b09b9cb8 100644 --- a/dolphinscheduler-server/src/main/resources/config/install_config.conf +++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf @@ -118,7 +118,7 @@ apiServerPort="12345" # install hosts # Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname -ips="ark0,ark1,ark2,ark3,ark4" +ips="ds1,ds2,ds3,ds4,ds5" # ssh port, default 22 # Note: if ssh port is not default, modify here @@ -126,19 +126,19 @@ sshPort="22" # run master machine # Note: list of hosts hostname for deploying master -masters="ark0,ark1" +masters="ds1,ds2" # run worker machine -# note: list of machine hostnames for deploying workers -workers="ark2,ark3,ark4" +# note: need to write the worker group name of each worker, the default value is "default" +workersGroup=(["ds1"]="default" ["ds2"]="default" ["ds3"]="default" ["ds4"]="default" ["ds5"]="default") # run alert machine # note: list of machine hostnames for deploying alert server -alertServer="ark3" +alertServer="ds3" # run api machine # note: list of machine hostnames for deploying api server -apiServers="ark1" +apiServers="ds1" # whether to start monitoring self-starting scripts monitorServerState="false" diff --git a/dolphinscheduler-server/src/main/resources/worker.properties b/dolphinscheduler-server/src/main/resources/worker.properties index d078f26ca6..36bc132743 100644 --- a/dolphinscheduler-server/src/main/resources/worker.properties +++ b/dolphinscheduler-server/src/main/resources/worker.properties @@ -21,7 +21,7 @@ # worker heartbeat interval #worker.heartbeat.interval=10 -# submit the number of tasks at a time +# submit the number of tasks at a time TODO #worker.fetch.task.num = 3 # only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 @@ -34,4 +34,4 @@ #worker.listen.port: 1234 # default worker group -#worker.group=default \ No newline at end of file +worker.group=default \ No newline at end of file diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh index f4949f311a..49c2409a6a 100644 --- a/script/scp-hosts.sh +++ b/script/scp-hosts.sh @@ -20,6 +20,12 @@ workDir=`dirname $0` workDir=`cd ${workDir};pwd` source $workDir/../conf/config/install_config.conf +txt="" +if [[ "$OSTYPE" == "darwin"* ]]; then + # Mac OSX + txt="''" +fi + hostsArr=(${ips//,/ }) for host in ${hostsArr[@]} do @@ -33,6 +39,11 @@ do for dsDir in bin conf lib script sql ui install.sh do + # if worker in workersGroup + if [[ "${map[${host}]}" ]] && [[ "${dsDir}" -eq "conf" ]]; then + sed -i ${txt} "s#worker.group.*#worker.group=${map[${host}]}#g" conf/worker.properties + fi + echo "start to scp $dsDir to $host/$installPath" scp -P $sshPort -r $workDir/../$dsDir $host:$installPath done diff --git a/script/start-all.sh b/script/start-all.sh index 11e4572059..5579a7d87e 100644 --- a/script/start-all.sh +++ b/script/start-all.sh @@ -28,8 +28,7 @@ do done -workersHost=(${workers//,/ }) -for worker in ${workersHost[@]} +for worker in ${!workersGroup[*]} do echo "$worker worker server is starting" diff --git a/script/stop-all.sh b/script/stop-all.sh index f761579cc8..e4ccf75a29 100644 --- a/script/stop-all.sh +++ b/script/stop-all.sh @@ -29,8 +29,7 @@ do done -workersHost=(${workers//,/ }) -for worker in ${workersHost[@]} +for worker in ${!workersGroup[*]} do echo "$worker worker server is stopping" ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;" From 43f1e8356fad45340d20a3cd92192844c65aa6e6 Mon Sep 17 00:00:00 2001 From: dailidong Date: Mon, 11 May 2020 19:38:40 +0800 Subject: [PATCH 27/43] add worker group config --- script/scp-hosts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh index 49c2409a6a..adee7d76c7 100644 --- a/script/scp-hosts.sh +++ b/script/scp-hosts.sh @@ -41,7 +41,7 @@ do do # if worker in workersGroup if [[ "${map[${host}]}" ]] && [[ "${dsDir}" -eq "conf" ]]; then - sed -i ${txt} "s#worker.group.*#worker.group=${map[${host}]}#g" conf/worker.properties + sed -i ${txt} "s#worker.group.*#worker.group=${map[${host}]}#g" $workDir/../conf/worker.properties fi echo "start to scp $dsDir to $host/$installPath" From ae14c1b3f63818c8d60263271ee4c24b7a82e974 Mon Sep 17 00:00:00 2001 From: dailidong Date: Wed, 13 May 2020 12:52:35 +0800 Subject: [PATCH 28/43] add worker group config --- .../src/main/resources/config/install_config.conf | 2 +- script/scp-hosts.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf index f6b09b9cb8..9a69306b2b 100644 --- a/dolphinscheduler-server/src/main/resources/config/install_config.conf +++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf @@ -130,7 +130,7 @@ masters="ds1,ds2" # run worker machine # note: need to write the worker group name of each worker, the default value is "default" -workersGroup=(["ds1"]="default" ["ds2"]="default" ["ds3"]="default" ["ds4"]="default" ["ds5"]="default") +declare -A workersGroup=(["ds1"]="default" ["ds2"]="default" ["ds3"]="default" ["ds4"]="default" ["ds5"]="default") # run alert machine # note: list of machine hostnames for deploying alert server diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh index adee7d76c7..da4f0d6cd5 100644 --- a/script/scp-hosts.sh +++ b/script/scp-hosts.sh @@ -40,8 +40,8 @@ do for dsDir in bin conf lib script sql ui install.sh do # if worker in workersGroup - if [[ "${map[${host}]}" ]] && [[ "${dsDir}" -eq "conf" ]]; then - sed -i ${txt} "s#worker.group.*#worker.group=${map[${host}]}#g" $workDir/../conf/worker.properties + if [[ "${workersGroup[${host}]}" ]] && [[ "${dsDir}" == "conf" ]]; then + sed -i ${txt} "s#worker.group.*#worker.group=${workersGroup[${host}]}#g" ${dsDir}/worker.properties fi echo "start to scp $dsDir to $host/$installPath" From 4aca1333f9c428ebaa3b3f02980d6dcf549f32fe Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 23 Jul 2020 23:32:00 +0800 Subject: [PATCH 29/43] =?UTF-8?q?this=20command=20just=20for=20getting=20a?= =?UTF-8?q?=20quick=20experience=EF=BC=8Cnot=20recommended=20for=20product?= =?UTF-8?q?ion.=20this=20operation=20will=20start=20a=20standalone=20zooke?= =?UTF-8?q?eper=20server?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/dolphinscheduler-daemon.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index b0437a30c2..31e252b6b0 100644 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -72,6 +72,10 @@ elif [ "$command" = "alert-server" ]; then CLASS=org.apache.dolphinscheduler.alert.AlertServer elif [ "$command" = "logger-server" ]; then CLASS=org.apache.dolphinscheduler.server.log.LoggerServer +elif [ "$command" = "zookeeper-server" ]; then + #note: this command just for getting a quick experience，not recommended for production. this operation will start a standalone zookeeper server + LOG_FILE="-Dlogback.configurationFile=classpath:logback-zookeeper.xml" + CLASS=org.apache.dolphinscheduler.service.zk.ZKServer else echo "Error: No command named \`$command' was found." exit 1 From 58d6a1ec9ab63cd482535572ea56d06431bfce48 Mon Sep 17 00:00:00 2001 From: dailidong Date: Fri, 24 Jul 2020 00:52:22 +0800 Subject: [PATCH 30/43] Merge remote-tracking branch 'upstream/dev' into dev --- .../main/resources/config/install_config.conf | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf index 517193a2eb..a08cb58ea0 100644 --- a/dolphinscheduler-server/src/main/resources/config/install_config.conf +++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf @@ -16,7 +16,7 @@ # -# NOTICE : If the following config has special characters in the variable `.*[]^${}\+?|()@#&`, Please escape, for example, `[` escape to `\[` +# NOTICE: If the following config has special characters in the variable `.*[]^${}\+?|()@#&`, Please escape, for example, `[` escape to `\[` # postgresql or mysql dbtype="mysql" @@ -27,12 +27,13 @@ dbhost="192.168.xx.xx:3306" # db username username="xx" +# db password +# NOTICE: if there are special characters, please use the \ to escape, for example, `[` escape to `\[` +password="xx" + # database name dbname="dolphinscheduler" -# db passwprd -# NOTICE: if there are special characters, please use the \ to escape, for example, `[` escape to `\[` -password="xx" # zk cluster zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181" @@ -64,15 +65,14 @@ mailUser="xxxxxxxxxx" mailPassword="xxxxxxxxxx" # TLS mail protocol support -starttlsEnable="false" - -#note: sslTrust is the same as mailServerHost -sslTrust="smtp.exmail.qq.com" +starttlsEnable="true" # SSL mail protocol support -# note: The SSL protocol is enabled by default. # only one of TLS and SSL can be in the true state. -sslEnable="true" +sslEnable="false" + +#note: sslTrust is the same as mailServerHost +sslTrust="smtp.exmail.qq.com" # resource storage type：HDFS,S3,NONE @@ -138,4 +138,4 @@ alertServer="ds3" # run api machine # note: list of machine hostnames for deploying api server -apiServers="ds1" +apiServers="ds1" \ No newline at end of file From 34082f1f77dca9c7229df4237f63e99090685327 Mon Sep 17 00:00:00 2001 From: dailidong Date: Fri, 24 Jul 2020 01:05:42 +0800 Subject: [PATCH 31/43] add logback for zookeeper --- .../src/main/resources/logback-zookeeper.xml | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 dolphinscheduler-service/src/main/resources/logback-zookeeper.xml diff --git a/dolphinscheduler-service/src/main/resources/logback-zookeeper.xml b/dolphinscheduler-service/src/main/resources/logback-zookeeper.xml new file mode 100644 index 0000000000..d92d0718c8 --- /dev/null +++ b/dolphinscheduler-service/src/main/resources/logback-zookeeper.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + + + + ${log.base}/dolphinscheduler-alert.log + + ${log.base}/dolphinscheduler-alert.%d{yyyy-MM-dd_HH}.%i.log + 20 + 64MB + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + + + + + + + + \ No newline at end of file From dea855f6cdc462a738d66a8a02b237218cfcaf85 Mon Sep 17 00:00:00 2001 From: dailidong Date: Fri, 24 Jul 2020 16:47:02 +0800 Subject: [PATCH 32/43] Update logback-zookeeper.xml --- .../src/main/resources/logback-zookeeper.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-service/src/main/resources/logback-zookeeper.xml b/dolphinscheduler-service/src/main/resources/logback-zookeeper.xml index d92d0718c8..34a15a7a5b 100644 --- a/dolphinscheduler-service/src/main/resources/logback-zookeeper.xml +++ b/dolphinscheduler-service/src/main/resources/logback-zookeeper.xml @@ -30,7 +30,7 @@ - ${log.base}/dolphinscheduler-alert.log + ${log.base}/dolphinscheduler-zookeeper.log ${log.base}/dolphinscheduler-alert.%d{yyyy-MM-dd_HH}.%i.log 20 @@ -49,4 +49,4 @@ - \ No newline at end of file + From d64127b90a5b5c8b4bcbdb8be56969c9d44e018e Mon Sep 17 00:00:00 2001 From: dailidong Date: Tue, 4 Aug 2020 17:57:23 +0800 Subject: [PATCH 33/43] optimize TheadUtils test --- .../common/threadutils/ThreadUtilsTest.java | 50 +++++++------------ 1 file changed, 18 insertions(+), 32 deletions(-) diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java index 2c76f40c0b..53441f0463 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java @@ -37,12 +37,12 @@ public class ThreadUtilsTest { @Test public void testNewDaemonFixedThreadExecutor() { // create core size and max size are all 3 - ExecutorService testExec = ThreadUtils.newDaemonFixedThreadExecutor("test-exec-thread",10); + ExecutorService testExec = ThreadUtils.newDaemonFixedThreadExecutor("test-exec-thread",3); - for (int i = 0; i < 19; i++) { + for (int i = 0; i < 2; i++) { final int index = i; testExec.submit(() -> { - System.out.println("do some work index " + index); + logger.info("do some work index " + index); }); } assertFalse(testExec.isShutdown()); @@ -77,13 +77,12 @@ public class ThreadUtilsTest { start.add(Calendar.SECOND, 1); globalTimer.add(Calendar.SECOND, 1); } - System.out.println("time is " + System.currentTimeMillis()); } }; scheduleService.scheduleAtFixedRate(schedulerTask, 2, 10, TimeUnit.SECONDS); assertFalse(scheduleService.isShutdown()); try { - Thread.sleep(60000); + Thread.sleep(2000); } catch (InterruptedException e) { e.printStackTrace(); } @@ -107,35 +106,22 @@ public class ThreadUtilsTest { */ @Test public void testThreadInfo() throws InterruptedException { - ThreadPoolExecutors workers = ThreadPoolExecutors.getInstance("worker", 3); - for (int i = 0; i < 5; ++i ) { + ThreadPoolExecutors workers = ThreadPoolExecutors.getInstance("worker", 1); + for (int i = 0; i < 2; ++i ) { int index = i; workers.execute(() -> { - for (int j = 0; j < 10; ++j) { + for (int j = 0; j < 1; ++j) { try { - Thread.sleep(1000); - System.out.printf("worker %d is doing the task", index); - System.out.println(); - workers.printStatus(); + Thread.sleep(100); + logger.info("worker %s is doing the task", index); +// workers.printStatus(); } catch (InterruptedException e) { - e.printStackTrace(); - } - } - }); - workers.submit(() -> { - for (int j = 0; j < 10; ++j) { - try { - Thread.sleep(1000); - System.out.printf("worker_2 %d is doing the task", index); - System.out.println(); - workers.printStatus(); - } catch (InterruptedException e) { - e.printStackTrace(); + logger.error("InterruptedException", e); } } }); } - Thread.sleep(50001); + Thread.sleep(1001); workers.shutdown(); } @@ -146,8 +132,8 @@ public class ThreadUtilsTest { public void testNewDaemonSingleThreadExecutor() { ExecutorService threadTest = ThreadUtils.newDaemonSingleThreadExecutor("thread_test"); threadTest.execute(() -> { - for (int i = 0; i < 100; ++i) { - System.out.println("daemon working "); + for (int i = 0; i < 2; ++i) { + logger.info("daemon working "); } }); @@ -161,8 +147,8 @@ public class ThreadUtilsTest { ThreadPoolExecutor threadPoolExecutor = ThreadUtils.newDaemonCachedThreadPool("threadTest-"); Thread thread1 = threadPoolExecutor.getThreadFactory().newThread(() -> { - for (int i = 0; i < 10; ++i) { - System.out.println("this task is with index " + i ); + for (int i = 0; i < 2; ++i) { + logger.info("this task is with index " + i ); } }); assertTrue(thread1.getName().startsWith("threadTest-")); @@ -174,10 +160,10 @@ public class ThreadUtilsTest { @Test public void testNewDaemonCachedThreadPoolWithThreadNumber() { ThreadPoolExecutor threadPoolExecutor = ThreadUtils.newDaemonCachedThreadPool("threadTest--", 3, 10); - for (int i = 0; i < 10; ++ i) { + for (int i = 0; i < 2; ++ i) { threadPoolExecutor.getThreadFactory().newThread(() -> { assertEquals(3, threadPoolExecutor.getActiveCount()); - System.out.println("this task is first work to do"); + logger.info("this task is first work to do"); }); } assertFalse(threadPoolExecutor.isShutdown()); From 6564e96315b1a9011ea9c97145450df5c59f318f Mon Sep 17 00:00:00 2001 From: dashi Date: Fri, 25 Sep 2020 17:51:28 +0800 Subject: [PATCH 34/43] [Fix][ui] Fix the problem that all pre-tasks are lost after dag reformatting #3823 --- dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js index d7f2f78a0c..74671cca57 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js @@ -135,10 +135,12 @@ Dag.prototype.backfill = function (arg) { const dataObject = {} g.nodes().forEach(function (v) { const node = g.node(v) + const location = store.state.dag.locations[node.label] const obj = {} - obj.name = node.label + obj.name = location.name obj.x = node.x + marginX obj.y = node.y + obj.targetarr = location.targetarr dataObject[node.label] = obj }) jsPlumb.ready(() => { From 8e0fb35366c08e5488a11e8ea5054da7f73c7cbc Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 8 Oct 2020 23:27:36 +0800 Subject: [PATCH 35/43] Update dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java Co-authored-by: Yichao Yang <1048262223@qq.com> --- .../dolphinscheduler/common/threadutils/ThreadUtilsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java index 53441f0463..821399ed6c 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java @@ -37,7 +37,7 @@ public class ThreadUtilsTest { @Test public void testNewDaemonFixedThreadExecutor() { // create core size and max size are all 3 - ExecutorService testExec = ThreadUtils.newDaemonFixedThreadExecutor("test-exec-thread",3); + ExecutorService testExec = ThreadUtils.newDaemonFixedThreadExecutor("test-exec-thread", 3); for (int i = 0; i < 2; i++) { final int index = i; From 13030502fd27863827ce9a2e3ec905c5a359170b Mon Sep 17 00:00:00 2001 From: BoYiZhang <39816903+BoYiZhang@users.noreply.github.com> Date: Thu, 15 Oct 2020 14:09:28 +0800 Subject: [PATCH 36/43] [FIX-3836][dev-API] process definition validation name interface prompt information error (#3908) * fix bug Delete invalid field: executorcores Modify verification prompt * fix bug Delete invalid field: executorcores Modify verification prompt * fix bug Delete invalid field: executorcores Modify verification prompt * dag add close button * reset last version * reset last version * dag add close buttion dag add close buttion * update CLICK_SAVE_WORKFLOW_BUTTON xpath * updae CLICK_SAVE_WORKFLOW_BUTTON xpath * updae CLICK_SAVE_WORKFLOW_BUTTON xpath * updae CLICK_SAVE_WORKFLOW_BUTTON xpath * Update CreateWorkflowLocator.java modify submit workflow button * Update CreateWorkflowLocator.java * Update CreateWorkflowLocator.java modify CLICK_ADD_BUTTON * Update CreateWorkflowLocator.java delete print * Update CreateWorkflowLocator.java 1 * Update CreateWorkflowLocator.java 1 * Setting '-XX:+DisableExplicitGC ' causes netty memory leaks in addition update '- XX: largepagesizeinbytes = 128M' to '- XX: largepagesizeinbytes = 10M' * Update dag.vue * Update dag.vue * Update dag.vue * Update CreateWorkflowLocator.java * Revert "Setting '-XX:+DisableExplicitGC ' causes netty memory leaks" This reverts commit 3a2cba7a * Setting '-XX:+DisableExplicitGC ' causes netty memory leaks in addition update '- XX: largepagesizeinbytes = 128M' to '- XX: largepagesizeinbytes = 10M' * Update dolphinscheduler-daemon.sh * process definition validation name interface prompt information error Co-authored-by: dailidong Co-authored-by: xingchun-chen <55787491+xingchun-chen@users.noreply.github.com> --- .../api/service/impl/ProcessDefinitionServiceImpl.java | 4 ++-- .../api/controller/ProcessDefinitionControllerTest.java | 4 ++-- .../api/service/ProcessDefinitionServiceTest.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index 90239b3969..8193093eb2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -223,7 +223,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements // return processDefinition object with ID result.put(Constants.DATA_LIST, processDefineMapper.selectById(processDefine.getId())); putMsg(result, Status.SUCCESS); - result.put("processDefinitionId", processDefine.getId()); + result.put(PROCESSDEFINITIONID, processDefine.getId()); return result; } @@ -438,7 +438,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements if (processDefinition == null) { putMsg(result, Status.SUCCESS); } else { - putMsg(result, Status.PROCESS_INSTANCE_EXIST, name); + putMsg(result, Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR, name); } return result; } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java index f2a54a1a88..692d28b56e 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java @@ -117,14 +117,14 @@ public class ProcessDefinitionControllerTest { public void testVerifyProcessDefinitionName() throws Exception { Map result = new HashMap<>(); - putMsg(result, Status.PROCESS_INSTANCE_EXIST); + putMsg(result, Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR); String projectName = "test"; String name = "dag_test"; Mockito.when(processDefinitionService.verifyProcessDefinitionName(user, projectName, name)).thenReturn(result); Result response = processDefinitionController.verifyProcessDefinitionName(user, projectName, name); - Assert.assertEquals(Status.PROCESS_INSTANCE_EXIST.getCode(), response.getCode().intValue()); + Assert.assertEquals(Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR.getCode(), response.getCode().intValue()); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index 9a9fcb0043..eb71cc87e7 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -627,7 +627,7 @@ public class ProcessDefinitionServiceTest { Mockito.when(processDefineMapper.verifyByDefineName(project.getId(), "test_pdf")).thenReturn(getProcessDefinition()); Map processExistRes = processDefinitionService.verifyProcessDefinitionName(loginUser, "project_test1", "test_pdf"); - Assert.assertEquals(Status.PROCESS_INSTANCE_EXIST, processExistRes.get(Constants.STATUS)); + Assert.assertEquals(Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR, processExistRes.get(Constants.STATUS)); } @Test From 2fbbf96cde705ff9799508c59f47135d31630ea7 Mon Sep 17 00:00:00 2001 From: zhuangchong <37063904+zhuangchong@users.noreply.github.com> Date: Thu, 15 Oct 2020 19:44:22 +0800 Subject: [PATCH 37/43] [Improvement-3686][service] ProcessService method name meets naming convention. (#3687) * ProcessService method name meets naming convention. * add code checkstyle. * update test. * update test. * add code checkstyle. * add code checkstyle. * update pom. * update pom. * ProcessService method name meets naming convention. * update processService code. * update code style. * update code style. * update code style. Co-authored-by: zhuangchong --- .../consumer/TaskPriorityQueueConsumer.java | 2 +- .../service/process/ProcessService.java | 403 +++++++++--------- 2 files changed, 196 insertions(+), 209 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java index 21995c3867..fb9f10d705 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java @@ -326,7 +326,7 @@ public class TaskPriorityQueueConsumer extends Thread{ udfFunIdsArray[i]=Integer.parseInt(udfFunIds[i]); } - List udfFuncList = processService.queryUdfFunListByids(udfFunIdsArray); + List udfFuncList = processService.queryUdfFunListByIds(udfFunIdsArray); Map udfFuncMap = new HashMap<>(); for(UdfFunc udfFunc : udfFuncList) { String tenantCode = processService.queryTenantCodeByResName(udfFunc.getResourceName(), ResourceType.UDF); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 7fca37470d..10069aea10 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.service.process; import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE; @@ -144,8 +145,6 @@ public class ProcessService { @Autowired private ResourceMapper resourceMapper; - - @Autowired private ErrorCommandMapper errorCommandMapper; @@ -167,12 +166,12 @@ public class ProcessService { public ProcessInstance handleCommand(Logger logger, String host, int validThreadNum, Command command) { ProcessInstance processInstance = constructProcessInstance(command, host); //cannot construct process instance, return null; - if(processInstance == null){ + if (processInstance == null) { logger.error("scan command, command parameter is error: {}", command); moveToErrorCommand(command, "process instance is null"); return null; } - if(!checkThreadNum(command, validThreadNum)){ + if (!checkThreadNum(command, validThreadNum)) { logger.info("there is not enough thread for this command: {}", command); return setWaitingThreadProcess(command, processInstance); } @@ -180,7 +179,7 @@ public class ProcessService { processInstance.addHistoryCmd(command.getCommandType()); saveProcessInstance(processInstance); this.setSubProcessParam(processInstance); - delCommandByid(command.getId()); + delCommandById(command.getId()); return processInstance; } @@ -193,7 +192,7 @@ public class ProcessService { public void moveToErrorCommand(Command command, String message) { ErrorCommand errorCommand = new ErrorCommand(command, message); this.errorCommandMapper.insert(errorCommand); - delCommandByid(command.getId()); + delCommandById(command.getId()); } /** @@ -204,7 +203,7 @@ public class ProcessService { */ private ProcessInstance setWaitingThreadProcess(Command command, ProcessInstance processInstance) { processInstance.setState(ExecutionStatus.WAITTING_THREAD); - if(command.getCommandType() != CommandType.RECOVER_WAITTING_THREAD){ + if (command.getCommandType() != CommandType.RECOVER_WAITTING_THREAD) { processInstance.addHistoryCmd(command.getCommandType()); } saveProcessInstance(processInstance); @@ -231,7 +230,7 @@ public class ProcessService { */ public int createCommand(Command command) { int result = 0; - if (command != null){ + if (command != null) { result = commandMapper.insert(command); } return result; @@ -241,7 +240,7 @@ public class ProcessService { * find one command from queue list * @return command */ - public Command findOneCommand(){ + public Command findOneCommand() { return commandMapper.getOneToRun(); } @@ -250,7 +249,7 @@ public class ProcessService { * @param command command * @return create command result */ - public Boolean verifyIsNeedCreateCommand(Command command){ + public Boolean verifyIsNeedCreateCommand(Command command) { Boolean isNeedCreate = true; Map cmdTypeMap = new HashMap(); cmdTypeMap.put(CommandType.REPEAT_RUNNING,1); @@ -258,16 +257,16 @@ public class ProcessService { cmdTypeMap.put(CommandType.START_FAILURE_TASK_PROCESS,1); CommandType commandType = command.getCommandType(); - if(cmdTypeMap.containsKey(commandType)){ + if (cmdTypeMap.containsKey(commandType)) { ObjectNode cmdParamObj = JSONUtils.parseObject(command.getCommandParam()); int processInstanceId = cmdParamObj.path(CMDPARAM_RECOVER_PROCESS_ID_STRING).asInt(); List commands = commandMapper.selectList(null); // for all commands - for (Command tmpCommand:commands){ - if(cmdTypeMap.containsKey(tmpCommand.getCommandType())){ + for (Command tmpCommand:commands) { + if (cmdTypeMap.containsKey(tmpCommand.getCommandType())) { ObjectNode tempObj = JSONUtils.parseObject(tmpCommand.getCommandParam()); - if(tempObj != null && processInstanceId == tempObj.path(CMDPARAM_RECOVER_PROCESS_ID_STRING).asInt()){ + if (tempObj != null && processInstanceId == tempObj.path(CMDPARAM_RECOVER_PROCESS_ID_STRING).asInt()) { isNeedCreate = false; break; } @@ -282,7 +281,7 @@ public class ProcessService { * @param processId processId * @return process instance */ - public ProcessInstance findProcessInstanceDetailById(int processId){ + public ProcessInstance findProcessInstanceDetailById(int processId) { return processInstanceMapper.queryDetailById(processId); } @@ -291,7 +290,7 @@ public class ProcessService { * @param defineId * @return */ - public List getTaskNodeListByDefinitionId(Integer defineId){ + public List getTaskNodeListByDefinitionId(Integer defineId) { ProcessDefinition processDefinition = processDefineMapper.selectById(defineId); if (processDefinition == null) { logger.info("process define not exists"); @@ -315,7 +314,7 @@ public class ProcessService { * @param processId processId * @return process instance */ - public ProcessInstance findProcessInstanceById(int processId){ + public ProcessInstance findProcessInstanceById(int processId) { return processInstanceMapper.selectById(processId); } @@ -333,7 +332,7 @@ public class ProcessService { * @param processInstanceId processInstanceId * @return delete process instance result */ - public int deleteWorkProcessInstanceById(int processInstanceId){ + public int deleteWorkProcessInstanceById(int processInstanceId) { return processInstanceMapper.deleteById(processInstanceId); } @@ -342,11 +341,11 @@ public class ProcessService { * @param processInstanceId processInstanceId * @return delete all sub process instance result */ - public int deleteAllSubWorkProcessByParentId(int processInstanceId){ + public int deleteAllSubWorkProcessByParentId(int processInstanceId) { List subProcessIdList = processInstanceMapMapper.querySubIdListByParentId(processInstanceId); - for(Integer subId : subProcessIdList){ + for (Integer subId : subProcessIdList) { deleteAllSubWorkProcessByParentId(subId); deleteWorkProcessMapByParentId(subId); removeTaskLogFile(subId); @@ -355,51 +354,48 @@ public class ProcessService { return 1; } - /** * remove task log file * @param processInstanceId processInstanceId */ - public void removeTaskLogFile(Integer processInstanceId){ + public void removeTaskLogFile(Integer processInstanceId) { LogClientService logClient = new LogClientService(); List taskInstanceList = findValidTaskListByProcessId(processInstanceId); - if (CollectionUtils.isEmpty(taskInstanceList)){ + if (CollectionUtils.isEmpty(taskInstanceList)) { return; } - for (TaskInstance taskInstance : taskInstanceList){ + for (TaskInstance taskInstance : taskInstanceList) { String taskLogPath = taskInstance.getLogPath(); - if (StringUtils.isEmpty(taskInstance.getHost())){ + if (StringUtils.isEmpty(taskInstance.getHost())) { continue; } int port = Constants.RPC_PORT; String ip = ""; try { ip = Host.of(taskInstance.getHost()).getIp(); - }catch (Exception e){ + } catch (Exception e) { // compatible old version ip = taskInstance.getHost(); } - // remove task log from loggerserver logClient.removeTaskLog(ip,port,taskLogPath); } } - /** * calculate sub process number in the process define. * @param processDefinitionId processDefinitionId * @return process thread num count */ - private Integer workProcessThreadNumCount(Integer processDefinitionId){ + private Integer workProcessThreadNumCount(Integer processDefinitionId) { List ids = new ArrayList<>(); recurseFindSubProcessId(processDefinitionId, ids); - return ids.size()+1; + return ids.size() + 1; } /** @@ -407,7 +403,7 @@ public class ProcessService { * @param parentId parentId * @param ids ids */ - public void recurseFindSubProcessId(int parentId, List ids){ + public void recurseFindSubProcessId(int parentId, List ids) { ProcessDefinition processDefinition = processDefineMapper.selectById(parentId); String processDefinitionJson = processDefinition.getProcessDefinitionJson(); @@ -415,12 +411,12 @@ public class ProcessService { List taskNodeList = processData.getTasks(); - if (taskNodeList != null && taskNodeList.size() > 0){ + if (taskNodeList != null && taskNodeList.size() > 0) { - for (TaskNode taskNode : taskNodeList){ + for (TaskNode taskNode : taskNodeList) { String parameter = taskNode.getParams(); ObjectNode parameterJson = JSONUtils.parseObject(parameter); - if (parameterJson.get(CMDPARAM_SUB_PROCESS_DEFINE_ID) != null){ + if (parameterJson.get(CMDPARAM_SUB_PROCESS_DEFINE_ID) != null) { SubProcessParameters subProcessParam = JSONUtils.parseObject(parameter, SubProcessParameters.class); ids.add(subProcessParam.getProcessDefinitionId()); recurseFindSubProcessId(subProcessParam.getProcessDefinitionId(),ids); @@ -441,8 +437,8 @@ public class ProcessService { public void createRecoveryWaitingThreadCommand(Command originCommand, ProcessInstance processInstance) { // sub process doesnot need to create wait command - if(processInstance.getIsSubProcess() == Flag.YES){ - if(originCommand != null){ + if (processInstance.getIsSubProcess() == Flag.YES) { + if (originCommand != null) { commandMapper.deleteById(originCommand.getId()); } return; @@ -450,7 +446,7 @@ public class ProcessService { Map cmdParam = new HashMap<>(); cmdParam.put(Constants.CMDPARAM_RECOVERY_WAITTING_THREAD, String.valueOf(processInstance.getId())); // process instance quit by "waiting thread" state - if(originCommand == null){ + if (originCommand == null) { Command command = new Command( CommandType.RECOVER_WAITTING_THREAD, processInstance.getTaskDependType(), @@ -464,14 +460,14 @@ public class ProcessService { processInstance.getProcessInstancePriority() ); saveCommand(command); - return ; + return; } // update the command time if current command if recover from waiting - if(originCommand.getCommandType() == CommandType.RECOVER_WAITTING_THREAD){ + if (originCommand.getCommandType() == CommandType.RECOVER_WAITTING_THREAD) { originCommand.setUpdateTime(new Date()); saveCommand(originCommand); - }else{ + } else { // delete old command and create new waiting thread command commandMapper.deleteById(originCommand.getId()); originCommand.setId(0); @@ -489,10 +485,10 @@ public class ProcessService { * @param cmdParam cmdParam map * @return date */ - private Date getScheduleTime(Command command, Map cmdParam){ + private Date getScheduleTime(Command command, Map cmdParam) { Date scheduleTime = command.getScheduleTime(); - if(scheduleTime == null){ - if(cmdParam != null && cmdParam.containsKey(CMDPARAM_COMPLEMENT_DATA_START_DATE)){ + if (scheduleTime == null) { + if (cmdParam != null && cmdParam.containsKey(CMDPARAM_COMPLEMENT_DATA_START_DATE)) { scheduleTime = DateUtils.stringToDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE)); } } @@ -508,7 +504,7 @@ public class ProcessService { */ private ProcessInstance generateNewProcessInstance(ProcessDefinition processDefinition, Command command, - Map cmdParam){ + Map cmdParam) { ProcessInstance processInstance = new ProcessInstance(processDefinition); processInstance.setState(ExecutionStatus.RUNNING_EXECUTION); processInstance.setRecovery(Flag.NO); @@ -529,7 +525,7 @@ public class ProcessService { // schedule time Date scheduleTime = getScheduleTime(command, cmdParam); - if(scheduleTime != null){ + if (scheduleTime != null) { processInstance.setScheduleTime(scheduleTime); } processInstance.setCommandStartTime(command.getStartTime()); @@ -562,17 +558,17 @@ public class ProcessService { * @param userId userId * @return tenant */ - public Tenant getTenantForProcess(int tenantId, int userId){ + public Tenant getTenantForProcess(int tenantId, int userId) { Tenant tenant = null; - if(tenantId >= 0){ + if (tenantId >= 0) { tenant = tenantMapper.queryById(tenantId); } - if (userId == 0){ + if (userId == 0) { return null; } - if(tenant == null){ + if (tenant == null) { User user = userMapper.selectById(userId); tenant = tenantMapper.queryById(user.getTenantId()); } @@ -585,11 +581,11 @@ public class ProcessService { * @param cmdParam cmdParam map * @return whether command param is valid */ - private Boolean checkCmdParam(Command command, Map cmdParam){ - if(command.getTaskDependType() == TaskDependType.TASK_ONLY || command.getTaskDependType()== TaskDependType.TASK_PRE){ - if(cmdParam == null + private Boolean checkCmdParam(Command command, Map cmdParam) { + if (command.getTaskDependType() == TaskDependType.TASK_ONLY || command.getTaskDependType() == TaskDependType.TASK_PRE) { + if (cmdParam == null || !cmdParam.containsKey(Constants.CMDPARAM_START_NODE_NAMES) - || cmdParam.get(Constants.CMDPARAM_START_NODE_NAMES).isEmpty()){ + || cmdParam.get(Constants.CMDPARAM_START_NODE_NAMES).isEmpty()) { logger.error("command node depend type is {}, but start nodes is null ", command.getTaskDependType()); return false; } @@ -603,78 +599,78 @@ public class ProcessService { * @param host host * @return process instance */ - private ProcessInstance constructProcessInstance(Command command, String host){ + private ProcessInstance constructProcessInstance(Command command, String host) { ProcessInstance processInstance = null; CommandType commandType = command.getCommandType(); Map cmdParam = JSONUtils.toMap(command.getCommandParam()); ProcessDefinition processDefinition = null; - if(command.getProcessDefinitionId() != 0){ + if (command.getProcessDefinitionId() != 0) { processDefinition = processDefineMapper.selectById(command.getProcessDefinitionId()); - if(processDefinition == null){ + if (processDefinition == null) { logger.error("cannot find the work process define! define id : {}", command.getProcessDefinitionId()); return null; } } - if(cmdParam != null ){ + if (cmdParam != null) { Integer processInstanceId = 0; // recover from failure or pause tasks - if(cmdParam.containsKey(Constants.CMDPARAM_RECOVER_PROCESS_ID_STRING)) { + if (cmdParam.containsKey(Constants.CMDPARAM_RECOVER_PROCESS_ID_STRING)) { String processId = cmdParam.get(Constants.CMDPARAM_RECOVER_PROCESS_ID_STRING); processInstanceId = Integer.parseInt(processId); if (processInstanceId == 0) { logger.error("command parameter is error, [ ProcessInstanceId ] is 0"); return null; } - }else if(cmdParam.containsKey(Constants.CMDPARAM_SUB_PROCESS)){ + } else if (cmdParam.containsKey(Constants.CMDPARAM_SUB_PROCESS)) { // sub process map String pId = cmdParam.get(Constants.CMDPARAM_SUB_PROCESS); processInstanceId = Integer.parseInt(pId); - }else if(cmdParam.containsKey(Constants.CMDPARAM_RECOVERY_WAITTING_THREAD)){ + } else if (cmdParam.containsKey(Constants.CMDPARAM_RECOVERY_WAITTING_THREAD)) { // waiting thread command String pId = cmdParam.get(Constants.CMDPARAM_RECOVERY_WAITTING_THREAD); processInstanceId = Integer.parseInt(pId); } - if(processInstanceId ==0){ + if (processInstanceId == 0) { processInstance = generateNewProcessInstance(processDefinition, command, cmdParam); - }else{ + } else { processInstance = this.findProcessInstanceDetailById(processInstanceId); } processDefinition = processDefineMapper.selectById(processInstance.getProcessDefinitionId()); processInstance.setProcessDefinition(processDefinition); //reset command parameter - if(processInstance.getCommandParam() != null){ + if (processInstance.getCommandParam() != null) { Map processCmdParam = JSONUtils.toMap(processInstance.getCommandParam()); - for(Map.Entry entry: processCmdParam.entrySet()) { - if(!cmdParam.containsKey(entry.getKey())){ + for (Map.Entry entry: processCmdParam.entrySet()) { + if (!cmdParam.containsKey(entry.getKey())) { cmdParam.put(entry.getKey(), entry.getValue()); } } } // reset command parameter if sub process - if(cmdParam.containsKey(Constants.CMDPARAM_SUB_PROCESS)){ + if (cmdParam.containsKey(Constants.CMDPARAM_SUB_PROCESS)) { processInstance.setCommandParam(command.getCommandParam()); } - }else{ + } else { // generate one new process instance processInstance = generateNewProcessInstance(processDefinition, command, cmdParam); } - if(!checkCmdParam(command, cmdParam)){ + if (!checkCmdParam(command, cmdParam)) { logger.error("command parameter check failed!"); return null; } - if(command.getScheduleTime() != null){ + if (command.getScheduleTime() != null) { processInstance.setScheduleTime(command.getScheduleTime()); } processInstance.setHost(host); ExecutionStatus runStatus = ExecutionStatus.RUNNING_EXECUTION; int runTime = processInstance.getRunTimes(); - switch (commandType){ + switch (commandType) { case START_PROCESS: break; case START_FAILURE_TASK_PROCESS: @@ -686,13 +682,13 @@ public class ProcessService { failedList.addAll(killedList); failedList.addAll(toleranceList); - for(Integer taskId : failedList){ + for (Integer taskId : failedList) { initTaskInstance(this.findTaskInstanceById(taskId)); } cmdParam.put(Constants.CMDPARAM_RECOVERY_START_NODE_STRING, String.join(Constants.COMMA, convertIntListToString(failedList))); processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam)); - processInstance.setRunTimes(runTime +1 ); + processInstance.setRunTimes(runTime + 1); break; case START_CURRENT_TASK_PROCESS: break; @@ -705,13 +701,13 @@ public class ProcessService { List stopNodeList = findTaskIdByInstanceState(processInstance.getId(), ExecutionStatus.KILL); suspendedNodeList.addAll(stopNodeList); - for(Integer taskId : suspendedNodeList){ + for (Integer taskId : suspendedNodeList) { // initialize the pause state initTaskInstance(this.findTaskInstanceById(taskId)); } cmdParam.put(Constants.CMDPARAM_RECOVERY_START_NODE_STRING, String.join(",", convertIntListToString(suspendedNodeList))); processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam)); - processInstance.setRunTimes(runTime +1); + processInstance.setRunTimes(runTime + 1); break; case RECOVER_TOLERANCE_FAULT_PROCESS: // recover tolerance fault process @@ -721,7 +717,7 @@ public class ProcessService { case COMPLEMENT_DATA: // delete all the valid tasks when complement data List taskInstanceList = this.findValidTaskListByProcessId(processInstance.getId()); - for(TaskInstance taskInstance : taskInstanceList){ + for (TaskInstance taskInstance : taskInstanceList) { taskInstance.setFlag(Flag.NO); this.updateTaskInstance(taskInstance); } @@ -729,19 +725,19 @@ public class ProcessService { break; case REPEAT_RUNNING: // delete the recover task names from command parameter - if(cmdParam.containsKey(Constants.CMDPARAM_RECOVERY_START_NODE_STRING)){ + if (cmdParam.containsKey(Constants.CMDPARAM_RECOVERY_START_NODE_STRING)) { cmdParam.remove(Constants.CMDPARAM_RECOVERY_START_NODE_STRING); processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam)); } // delete all the valid tasks when repeat running List validTaskList = findValidTaskListByProcessId(processInstance.getId()); - for(TaskInstance taskInstance : validTaskList){ + for (TaskInstance taskInstance : validTaskList) { taskInstance.setFlag(Flag.NO); updateTaskInstance(taskInstance); } processInstance.setStartTime(new Date()); processInstance.setEndTime(null); - processInstance.setRunTimes(runTime +1); + processInstance.setRunTimes(runTime + 1); initComplementDataParam(processDefinition, processInstance, cmdParam); break; case SCHEDULER: @@ -759,10 +755,10 @@ public class ProcessService { * @param command command * @return command type */ - private CommandType getCommandTypeIfComplement(ProcessInstance processInstance, Command command){ - if(CommandType.COMPLEMENT_DATA == processInstance.getCmdTypeIfComplement()){ + private CommandType getCommandTypeIfComplement(ProcessInstance processInstance, Command command) { + if (CommandType.COMPLEMENT_DATA == processInstance.getCmdTypeIfComplement()) { return CommandType.COMPLEMENT_DATA; - }else{ + } else { return command.getCommandType(); } } @@ -776,13 +772,13 @@ public class ProcessService { private void initComplementDataParam(ProcessDefinition processDefinition, ProcessInstance processInstance, Map cmdParam) { - if(!processInstance.isComplementData()){ + if (!processInstance.isComplementData()) { return; } Date startComplementTime = DateUtils.parse(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE), YYYY_MM_DD_HH_MM_SS); - if(Flag.NO == processInstance.getIsSubProcess()) { + if (Flag.NO == processInstance.getIsSubProcess()) { processInstance.setScheduleTime(startComplementTime); } processInstance.setGlobalParams(ParameterUtils.curingGlobalParams( @@ -792,7 +788,6 @@ public class ProcessService { } - /** * set sub work process parameters. * handle sub work process instance, update relation table and command parameters @@ -800,15 +795,15 @@ public class ProcessService { * @param subProcessInstance subProcessInstance * @return process instance */ - public ProcessInstance setSubProcessParam(ProcessInstance subProcessInstance){ + public ProcessInstance setSubProcessParam(ProcessInstance subProcessInstance) { String cmdParam = subProcessInstance.getCommandParam(); - if(StringUtils.isEmpty(cmdParam)){ + if (StringUtils.isEmpty(cmdParam)) { return subProcessInstance; } Map paramMap = JSONUtils.toMap(cmdParam); // write sub process id into cmd param. - if(paramMap.containsKey(CMDPARAM_SUB_PROCESS) - && CMDPARAM_EMPTY_SUB_PROCESS.equals(paramMap.get(CMDPARAM_SUB_PROCESS))){ + if (paramMap.containsKey(CMDPARAM_SUB_PROCESS) + && CMDPARAM_EMPTY_SUB_PROCESS.equals(paramMap.get(CMDPARAM_SUB_PROCESS))) { paramMap.remove(CMDPARAM_SUB_PROCESS); paramMap.put(CMDPARAM_SUB_PROCESS, String.valueOf(subProcessInstance.getId())); subProcessInstance.setCommandParam(JSONUtils.toJsonString(paramMap)); @@ -817,18 +812,18 @@ public class ProcessService { } // copy parent instance user def params to sub process.. String parentInstanceId = paramMap.get(CMDPARAM_SUB_PROCESS_PARENT_INSTANCE_ID); - if(StringUtils.isNotEmpty(parentInstanceId)){ + if (StringUtils.isNotEmpty(parentInstanceId)) { ProcessInstance parentInstance = findProcessInstanceDetailById(Integer.parseInt(parentInstanceId)); - if(parentInstance != null){ + if (parentInstance != null) { subProcessInstance.setGlobalParams( joinGlobalParams(parentInstance.getGlobalParams(), subProcessInstance.getGlobalParams())); this.saveProcessInstance(subProcessInstance); - }else{ + } else { logger.error("sub process command params error, cannot find parent instance: {} ", cmdParam); } } ProcessInstanceMap processInstanceMap = JSONUtils.parseObject(cmdParam, ProcessInstanceMap.class); - if(processInstanceMap == null || processInstanceMap.getParentProcessInstanceId() == 0){ + if (processInstanceMap == null || processInstanceMap.getParentProcessInstanceId() == 0) { return subProcessInstance; } // update sub process id to process map table @@ -845,15 +840,15 @@ public class ProcessService { * @param subGlobalParams subGlobalParams * @return global params join */ - private String joinGlobalParams(String parentGlobalParams, String subGlobalParams){ + private String joinGlobalParams(String parentGlobalParams, String subGlobalParams) { List parentPropertyList = JSONUtils.toList(parentGlobalParams, Property.class); List subPropertyList = JSONUtils.toList(subGlobalParams, Property.class); Map subMap = subPropertyList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); - for(Property parent : parentPropertyList){ - if(!subMap.containsKey(parent.getProp())){ + for (Property parent : parentPropertyList) { + if (!subMap.containsKey(parent.getProp())) { subPropertyList.add(parent); } } @@ -864,10 +859,10 @@ public class ProcessService { * initialize task instance * @param taskInstance taskInstance */ - private void initTaskInstance(TaskInstance taskInstance){ + private void initTaskInstance(TaskInstance taskInstance) { - if(!taskInstance.isSubProcess()){ - if(taskInstance.getState().typeIsCancel() || taskInstance.getState().typeIsFailure()){ + if (!taskInstance.isSubProcess()) { + if (taskInstance.getState().typeIsCancel() || taskInstance.getState().typeIsFailure()) { taskInstance.setFlag(Flag.NO); updateTaskInstance(taskInstance); return; @@ -884,18 +879,18 @@ public class ProcessService { * @return task instance */ @Transactional(rollbackFor = RuntimeException.class) - public TaskInstance submitTask(TaskInstance taskInstance){ + public TaskInstance submitTask(TaskInstance taskInstance) { ProcessInstance processInstance = this.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); logger.info("start submit task : {}, instance id:{}, state: {}", taskInstance.getName(), taskInstance.getProcessInstanceId(), processInstance.getState()); //submit to db TaskInstance task = submitTaskInstanceToDB(taskInstance, processInstance); - if(task == null){ + if (task == null) { logger.error("end submit task to db error, task name:{}, process id:{} state: {} ", taskInstance.getName(), taskInstance.getProcessInstance(), processInstance.getState()); return task; } - if(!task.getState().typeIsFinished()){ + if (!task.getState().typeIsFinished()) { createSubWorkProcess(processInstance, task); } @@ -913,7 +908,7 @@ public class ProcessService { * @param parentTask parentTask * @return process instance map */ - private ProcessInstanceMap setProcessInstanceMap(ProcessInstance parentInstance, TaskInstance parentTask){ + private ProcessInstanceMap setProcessInstanceMap(ProcessInstance parentInstance, TaskInstance parentTask) { ProcessInstanceMap processMap = findWorkProcessMapByParent(parentInstance.getId(), parentTask.getId()); if (processMap != null) { return processMap; @@ -1077,7 +1072,7 @@ public class ProcessService { private void updateSubProcessDefinitionByParent(ProcessInstance parentProcessInstance, int childDefinitionId) { ProcessDefinition fatherDefinition = this.findProcessDefineById(parentProcessInstance.getProcessDefinitionId()); ProcessDefinition childDefinition = this.findProcessDefineById(childDefinitionId); - if(childDefinition != null && fatherDefinition != null){ + if (childDefinition != null && fatherDefinition != null) { childDefinition.setReceivers(fatherDefinition.getReceivers()); childDefinition.setReceiversCc(fatherDefinition.getReceiversCc()); processDefineMapper.updateById(childDefinition); @@ -1090,22 +1085,22 @@ public class ProcessService { * @param processInstance processInstance * @return task instance */ - public TaskInstance submitTaskInstanceToDB(TaskInstance taskInstance, ProcessInstance processInstance){ + public TaskInstance submitTaskInstanceToDB(TaskInstance taskInstance, ProcessInstance processInstance) { ExecutionStatus processInstanceState = processInstance.getState(); - if(taskInstance.getState().typeIsFailure()){ - if(taskInstance.isSubProcess()){ - taskInstance.setRetryTimes(taskInstance.getRetryTimes() + 1 ); - }else { + if (taskInstance.getState().typeIsFailure()) { + if (taskInstance.isSubProcess()) { + taskInstance.setRetryTimes(taskInstance.getRetryTimes() + 1); + } else { - if( processInstanceState != ExecutionStatus.READY_STOP - && processInstanceState != ExecutionStatus.READY_PAUSE){ + if (processInstanceState != ExecutionStatus.READY_STOP + && processInstanceState != ExecutionStatus.READY_PAUSE) { // failure task set invalid taskInstance.setFlag(Flag.NO); updateTaskInstance(taskInstance); // crate new task instance - if(taskInstance.getState() != ExecutionStatus.NEED_FAULT_TOLERANCE){ - taskInstance.setRetryTimes(taskInstance.getRetryTimes() + 1 ); + if (taskInstance.getState() != ExecutionStatus.NEED_FAULT_TOLERANCE) { + taskInstance.setRetryTimes(taskInstance.getRetryTimes() + 1); } taskInstance.setSubmitTime(null); taskInstance.setStartTime(null); @@ -1126,13 +1121,12 @@ public class ProcessService { taskInstance.setFirstSubmitTime(taskInstance.getSubmitTime()); } boolean saveResult = saveTaskInstance(taskInstance); - if(!saveResult){ + if (!saveResult) { return null; } return taskInstance; } - /** * ${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskInstanceId}_${task executed by ip1},${ip2}... * The tasks with the highest priority are selected by comparing the priorities of the above four levels from high to low. @@ -1143,7 +1137,7 @@ public class ProcessService { String taskWorkerGroup = getTaskWorkerGroup(taskInstance); ProcessInstance processInstance = this.findProcessInstanceById(taskInstance.getProcessInstanceId()); - if(processInstance == null){ + if (processInstance == null) { logger.error("process instance is null. please check the task info, task id: " + taskInstance.getId()); return ""; } @@ -1171,9 +1165,9 @@ public class ProcessService { * @param processInstanceState processInstanceState * @return process instance state */ - public ExecutionStatus getSubmitTaskState(TaskInstance taskInstance, ExecutionStatus processInstanceState){ + public ExecutionStatus getSubmitTaskState(TaskInstance taskInstance, ExecutionStatus processInstanceState) { ExecutionStatus state = taskInstance.getState(); - if( + if ( // running, delayed or killed // the task already exists in task queue // return state @@ -1181,17 +1175,17 @@ public class ProcessService { || state == ExecutionStatus.DELAY_EXECUTION || state == ExecutionStatus.KILL || checkTaskExistsInTaskQueue(taskInstance) - ){ + ) { return state; } //return pasue /stop if process instance state is ready pause / stop // or return submit success - if( processInstanceState == ExecutionStatus.READY_PAUSE){ + if (processInstanceState == ExecutionStatus.READY_PAUSE) { state = ExecutionStatus.PAUSE; - }else if(processInstanceState == ExecutionStatus.READY_STOP + } else if (processInstanceState == ExecutionStatus.READY_STOP || !checkProcessStrategy(taskInstance)) { state = ExecutionStatus.KILL; - }else{ + } else { state = ExecutionStatus.SUBMITTED_SUCCESS; } return state; @@ -1202,16 +1196,16 @@ public class ProcessService { * @param taskInstance taskInstance * @return check strategy result */ - private boolean checkProcessStrategy(TaskInstance taskInstance){ + private boolean checkProcessStrategy(TaskInstance taskInstance) { ProcessInstance processInstance = this.findProcessInstanceById(taskInstance.getProcessInstanceId()); FailureStrategy failureStrategy = processInstance.getFailureStrategy(); - if(failureStrategy == FailureStrategy.CONTINUE){ + if (failureStrategy == FailureStrategy.CONTINUE) { return true; } List taskInstances = this.findValidTaskListByProcessId(taskInstance.getProcessInstanceId()); - for(TaskInstance task : taskInstances){ - if(task.getState() == ExecutionStatus.FAILURE){ + for (TaskInstance task : taskInstances) { + if (task.getState() == ExecutionStatus.FAILURE) { return false; } } @@ -1223,8 +1217,8 @@ public class ProcessService { * @param taskInstance taskInstance * @return whether taskinstance exists queue */ - public boolean checkTaskExistsInTaskQueue(TaskInstance taskInstance){ - if(taskInstance.isSubProcess()){ + public boolean checkTaskExistsInTaskQueue(TaskInstance taskInstance) { + if (taskInstance.isSubProcess()) { return false; } @@ -1237,9 +1231,9 @@ public class ProcessService { * create a new process instance * @param processInstance processInstance */ - public void createProcessInstance(ProcessInstance processInstance){ + public void createProcessInstance(ProcessInstance processInstance) { - if (processInstance != null){ + if (processInstance != null) { processInstanceMapper.insert(processInstance); } } @@ -1248,15 +1242,15 @@ public class ProcessService { * insert or update work process instance to data base * @param processInstance processInstance */ - public void saveProcessInstance(ProcessInstance processInstance){ + public void saveProcessInstance(ProcessInstance processInstance) { - if (processInstance == null){ + if (processInstance == null) { logger.error("save error, process instance is null!"); - return ; + return; } - if(processInstance.getId() != 0){ + if (processInstance.getId() != 0) { processInstanceMapper.updateById(processInstance); - }else{ + } else { createProcessInstance(processInstance); } } @@ -1266,10 +1260,10 @@ public class ProcessService { * @param command command * @return save command result */ - public int saveCommand(Command command){ - if(command.getId() != 0){ + public int saveCommand(Command command) { + if (command.getId() != 0) { return commandMapper.updateById(command); - }else{ + } else { return commandMapper.insert(command); } } @@ -1279,10 +1273,10 @@ public class ProcessService { * @param taskInstance taskInstance * @return save task instance result */ - public boolean saveTaskInstance(TaskInstance taskInstance){ - if(taskInstance.getId() != 0){ + public boolean saveTaskInstance(TaskInstance taskInstance) { + if (taskInstance.getId() != 0) { return updateTaskInstance(taskInstance); - }else{ + } else { return createTaskInstance(taskInstance); } } @@ -1302,15 +1296,16 @@ public class ProcessService { * @param taskInstance taskInstance * @return update task instance result */ - public boolean updateTaskInstance(TaskInstance taskInstance){ + public boolean updateTaskInstance(TaskInstance taskInstance) { int count = taskInstanceMapper.updateById(taskInstance); return count > 0; } + /** * delete a command by id * @param id id */ - public void delCommandByid(int id) { + public void delCommandById(int id) { commandMapper.deleteById(id); } @@ -1319,20 +1314,19 @@ public class ProcessService { * @param taskId task id * @return task intance */ - public TaskInstance findTaskInstanceById(Integer taskId){ + public TaskInstance findTaskInstanceById(Integer taskId) { return taskInstanceMapper.selectById(taskId); } - /** * package task instance，associate processInstance and processDefine * @param taskInstId taskInstId * @return task instance */ - public TaskInstance getTaskInstanceDetailByTaskId(int taskInstId){ + public TaskInstance getTaskInstanceDetailByTaskId(int taskInstId) { // get task instance TaskInstance taskInstance = findTaskInstanceById(taskInstId); - if(taskInstance == null){ + if (taskInstance == null) { return taskInstance; } // get process instance @@ -1345,14 +1339,13 @@ public class ProcessService { return taskInstance; } - /** * get id list by task state * @param instanceId instanceId * @param state state * @return task instance states */ - public List findTaskIdByInstanceState(int instanceId, ExecutionStatus state){ + public List findTaskIdByInstanceState(int instanceId, ExecutionStatus state) { return taskInstanceMapper.queryTaskByProcessIdAndState(instanceId, state.ordinal()); } @@ -1361,8 +1354,8 @@ public class ProcessService { * @param processInstanceId processInstanceId * @return task instance list */ - public List findValidTaskListByProcessId(Integer processInstanceId){ - return taskInstanceMapper.findValidTaskListByProcessId(processInstanceId, Flag.YES); + public List findValidTaskListByProcessId(Integer processInstanceId) { + return taskInstanceMapper.findValidTaskListByProcessId(processInstanceId, Flag.YES); } /** @@ -1370,7 +1363,7 @@ public class ProcessService { * @param processInstanceId processInstanceId * @return task instance list */ - public List findPreviousTaskListByWorkProcessId(Integer processInstanceId){ + public List findPreviousTaskListByWorkProcessId(Integer processInstanceId) { return taskInstanceMapper.findValidTaskListByProcessId(processInstanceId, Flag.NO); } @@ -1379,19 +1372,18 @@ public class ProcessService { * @param processInstanceMap processInstanceMap * @return update process instance result */ - public int updateWorkProcessInstanceMap(ProcessInstanceMap processInstanceMap){ + public int updateWorkProcessInstanceMap(ProcessInstanceMap processInstanceMap) { return processInstanceMapMapper.updateById(processInstanceMap); } - /** * create work process instance map * @param processInstanceMap processInstanceMap * @return create process instance result */ - public int createWorkProcessInstanceMap(ProcessInstanceMap processInstanceMap){ + public int createWorkProcessInstanceMap(ProcessInstanceMap processInstanceMap) { Integer count = 0; - if(processInstanceMap !=null){ + if (processInstanceMap != null) { return processInstanceMapMapper.insert(processInstanceMap); } return count; @@ -1403,7 +1395,7 @@ public class ProcessService { * @param parentTaskId parentTaskId * @return process instance map */ - public ProcessInstanceMap findWorkProcessMapByParent(Integer parentWorkProcessId, Integer parentTaskId){ + public ProcessInstanceMap findWorkProcessMapByParent(Integer parentWorkProcessId, Integer parentTaskId) { return processInstanceMapMapper.queryByParentId(parentWorkProcessId, parentTaskId); } @@ -1412,7 +1404,7 @@ public class ProcessService { * @param parentWorkProcessId parentWorkProcessId * @return delete process map result */ - public int deleteWorkProcessMapByParentId(int parentWorkProcessId){ + public int deleteWorkProcessMapByParentId(int parentWorkProcessId) { return processInstanceMapMapper.deleteByParentProcessId(parentWorkProcessId); } @@ -1423,10 +1415,10 @@ public class ProcessService { * @param parentTaskId parentTaskId * @return process instance */ - public ProcessInstance findSubProcessInstance(Integer parentProcessId, Integer parentTaskId){ + public ProcessInstance findSubProcessInstance(Integer parentProcessId, Integer parentTaskId) { ProcessInstance processInstance = null; ProcessInstanceMap processInstanceMap = processInstanceMapMapper.queryByParentId(parentProcessId, parentTaskId); - if(processInstanceMap == null || processInstanceMap.getProcessInstanceId() == 0){ + if (processInstanceMap == null || processInstanceMap.getProcessInstanceId() == 0) { return processInstance; } processInstance = findProcessInstanceById(processInstanceMap.getProcessInstanceId()); @@ -1441,14 +1433,13 @@ public class ProcessService { public ProcessInstance findParentProcessInstance(Integer subProcessId) { ProcessInstance processInstance = null; ProcessInstanceMap processInstanceMap = processInstanceMapMapper.queryBySubProcessId(subProcessId); - if(processInstanceMap == null || processInstanceMap.getProcessInstanceId() == 0){ + if (processInstanceMap == null || processInstanceMap.getProcessInstanceId() == 0) { return processInstance; } processInstance = findProcessInstanceById(processInstanceMap.getParentProcessInstanceId()); return processInstance; } - /** * change task state * @param state state @@ -1476,7 +1467,7 @@ public class ProcessService { * @param processInstance processInstance * @return update process instance result */ - public int updateProcessInstance(ProcessInstance processInstance){ + public int updateProcessInstance(ProcessInstance processInstance) { return processInstanceMapper.updateById(processInstance); } @@ -1493,9 +1484,9 @@ public class ProcessService { */ public int updateProcessInstance(Integer processInstanceId, String processJson, String globalParams, Date scheduleTime, Flag flag, - String locations, String connects){ + String locations, String connects) { ProcessInstance processInstance = processInstanceMapper.queryDetailById(processInstanceId); - if(processInstance!= null){ + if (processInstance != null) { processInstance.setProcessInstanceJson(processJson); processInstance.setGlobalParams(globalParams); processInstance.setScheduleTime(scheduleTime); @@ -1533,12 +1524,12 @@ public class ProcessService { * @param intList intList * @return string list */ - public List convertIntListToString(List intList){ - if(intList == null){ + public List convertIntListToString(List intList) { + if (intList == null) { return new ArrayList<>(); } List result = new ArrayList(intList.size()); - for(Integer intVar : intList){ + for (Integer intVar : intList) { result.add(String.valueOf(intVar)); } return result; @@ -1567,8 +1558,7 @@ public class ProcessService { * @param host host * @return process instance list */ - public List queryNeedFailoverProcessInstances(String host){ - + public List queryNeedFailoverProcessInstances(String host) { return processInstanceMapper.queryByHostAndStatus(host, stateArray); } @@ -1577,7 +1567,7 @@ public class ProcessService { * @param processInstance processInstance */ @Transactional(rollbackFor = RuntimeException.class) - public void processNeedFailoverProcessInstances(ProcessInstance processInstance){ + public void processNeedFailoverProcessInstances(ProcessInstance processInstance) { //1 update processInstance host is null processInstance.setHost(Constants.NULL); processInstanceMapper.updateById(processInstance); @@ -1596,7 +1586,7 @@ public class ProcessService { * @param host host * @return task instance list */ - public List queryNeedFailoverTaskInstances(String host){ + public List queryNeedFailoverTaskInstances(String host) { return taskInstanceMapper.queryByHostAndStatus(host, stateArray); } @@ -1606,11 +1596,10 @@ public class ProcessService { * @param id id * @return datasource */ - public DataSource findDataSourceById(int id){ + public DataSource findDataSourceById(int id) { return dataSourceMapper.selectById(id); } - /** * update process instance state by id * @param processInstanceId processInstanceId @@ -1629,9 +1618,9 @@ public class ProcessService { * @param taskId taskId * @return process instance */ - public ProcessInstance findProcessInstanceByTaskId(int taskId){ + public ProcessInstance findProcessInstanceByTaskId(int taskId) { TaskInstance taskInstance = taskInstanceMapper.selectById(taskId); - if(taskInstance!= null){ + if (taskInstance != null) { return processInstanceMapper.selectById(taskInstance.getProcessInstanceId()); } return null; @@ -1642,7 +1631,7 @@ public class ProcessService { * @param ids ids * @return udf function list */ - public List queryUdfFunListByids(int[] ids){ + public List queryUdfFunListByIds(int[] ids) { return udfFuncMapper.queryUdfByIdStr(ids, null); } @@ -1652,7 +1641,7 @@ public class ProcessService { * @param resourceType resource type * @return tenant code */ - public String queryTenantCodeByResName(String resName,ResourceType resourceType){ + public String queryTenantCodeByResName(String resName,ResourceType resourceType) { return resourceMapper.queryTenantCodeByResourceName(resName, resourceType.ordinal()); } @@ -1661,7 +1650,7 @@ public class ProcessService { * @param ids ids * @return schedule list */ - public List selectAllByProcessDefineId(int[] ids){ + public List selectAllByProcessDefineId(int[] ids) { return scheduleMapper.selectAllByProcessDefineArray( ids); } @@ -1676,7 +1665,7 @@ public class ProcessService { */ public CycleDependency getCycleDependency(int masterId, int processDefinitionId, Date scheduledFireTime) throws Exception { List list = getCycleDependencies(masterId,new int[]{processDefinitionId},scheduledFireTime); - return list.size()>0 ? list.get(0) : null; + return list.size() > 0 ? list.get(0) : null; } @@ -1694,29 +1683,28 @@ public class ProcessService { logger.warn("ids[] is empty!is invalid!"); return cycleDependencyList; } - if(scheduledFireTime == null){ + if (scheduledFireTime == null) { logger.warn("scheduledFireTime is null!is invalid!"); return cycleDependencyList; } - String strCrontab = ""; CronExpression depCronExpression; Cron depCron; List list; List schedules = this.selectAllByProcessDefineId(ids); // for all scheduling information - for(Schedule depSchedule:schedules){ + for (Schedule depSchedule:schedules) { strCrontab = depSchedule.getCrontab(); depCronExpression = CronUtils.parse2CronExpression(strCrontab); depCron = CronUtils.parse2Cron(strCrontab); CycleEnum cycleEnum = CronUtils.getMiniCycle(depCron); - if(cycleEnum == null){ + if (cycleEnum == null) { logger.error("{} is not valid",strCrontab); continue; } Calendar calendar = Calendar.getInstance(); - switch (cycleEnum){ + switch (cycleEnum) { /*case MINUTE: calendar.add(Calendar.MINUTE,-61);*/ case HOUR: @@ -1737,13 +1725,13 @@ public class ProcessService { } Date start = calendar.getTime(); - if(depSchedule.getProcessDefinitionId() == masterId){ + if (depSchedule.getProcessDefinitionId() == masterId) { list = CronUtils.getSelfFireDateList(start, scheduledFireTime, depCronExpression); - }else { + } else { list = CronUtils.getFireDateList(start, scheduledFireTime, depCronExpression); } - if(list.size()>=1){ - start = list.get(list.size()-1); + if (list.size() >= 1) { + start = list.get(list.size() - 1); CycleDependency dependency = new CycleDependency(depSchedule.getProcessDefinitionId(),start, CronUtils.getExpirationTime(start, cycleEnum), cycleEnum); cycleDependencyList.add(dependency); } @@ -1795,22 +1783,20 @@ public class ProcessService { * @param processInstanceId processInstanceId * @return queue */ - public String queryUserQueueByProcessInstanceId(int processInstanceId){ + public String queryUserQueueByProcessInstanceId(int processInstanceId) { String queue = ""; ProcessInstance processInstance = processInstanceMapper.selectById(processInstanceId); - if(processInstance == null){ + if (processInstance == null) { return queue; } User executor = userMapper.selectById(processInstance.getExecutorId()); - if(executor != null){ + if (executor != null) { queue = executor.getQueue(); } return queue; } - - /** * get task worker group * @param taskInstance taskInstance @@ -1819,13 +1805,13 @@ public class ProcessService { public String getTaskWorkerGroup(TaskInstance taskInstance) { String workerGroup = taskInstance.getWorkerGroup(); - if(StringUtils.isNotBlank(workerGroup)){ + if (StringUtils.isNotBlank(workerGroup)) { return workerGroup; } int processInstanceId = taskInstance.getProcessInstanceId(); ProcessInstance processInstance = findProcessInstanceById(processInstanceId); - if(processInstance != null){ + if (processInstance != null) { return processInstance.getWorkerGroup(); } logger.info("task : {} will use default worker group", taskInstance.getId()); @@ -1837,15 +1823,15 @@ public class ProcessService { * @param userId userId * @return project list */ - public List getProjectListHavePerm(int userId){ + public List getProjectListHavePerm(int userId) { List createProjects = projectMapper.queryProjectCreatedByUser(userId); List authedProjects = projectMapper.queryAuthedProjectListByUserId(userId); - if(createProjects == null){ + if (createProjects == null) { createProjects = new ArrayList<>(); } - if(authedProjects != null){ + if (authedProjects != null) { createProjects.addAll(authedProjects); } return createProjects; @@ -1856,10 +1842,10 @@ public class ProcessService { * @param userId userId * @return project ids */ - public List getProjectIdListHavePerm(int userId){ + public List getProjectIdListHavePerm(int userId) { List projectIdList = new ArrayList<>(); - for(Project project : getProjectListHavePerm(userId)){ + for (Project project : getProjectListHavePerm(userId)) { projectIdList.add(project.getId()); } return projectIdList; @@ -1871,13 +1857,13 @@ public class ProcessService { * @param needChecks data source id array * @return unauthorized udf function list */ - public List listUnauthorized(int userId,T[] needChecks,AuthorizationType authorizationType){ + public List listUnauthorized(int userId,T[] needChecks,AuthorizationType authorizationType) { List resultList = new ArrayList(); if (Objects.nonNull(needChecks) && needChecks.length > 0) { Set originResSet = new HashSet(Arrays.asList(needChecks)); - switch (authorizationType){ + switch (authorizationType) { case RESOURCE_FILE_ID: Set authorizedResourceFiles = resourceMapper.listAuthorizedResourceById(userId, needChecks).stream().map(t -> t.getId()).collect(toSet()); originResSet.removeAll(authorizedResourceFiles); @@ -1898,6 +1884,8 @@ public class ProcessService { Set authorizedUdfs = udfFuncMapper.listAuthorizedUdfFunc(userId, needChecks).stream().map(t -> t.getId()).collect(toSet()); originResSet.removeAll(authorizedUdfs); break; + default: + break; } resultList.addAll(originResSet); @@ -1911,7 +1899,7 @@ public class ProcessService { * @param userId user id * @return User */ - public User getUserById(int userId){ + public User getUserById(int userId) { return userMapper.selectById(userId); } @@ -1920,17 +1908,16 @@ public class ProcessService { * @param resoruceId resource id * @return Resource */ - public Resource getResourceById(int resoruceId){ + public Resource getResourceById(int resoruceId) { return resourceMapper.selectById(resoruceId); } - /** * list resources by ids * @param resIds resIds * @return resource list */ - public List listResourceByIds(Integer[] resIds){ + public List listResourceByIds(Integer[] resIds) { return resourceMapper.listResourceByIds(resIds); } @@ -1939,11 +1926,11 @@ public class ProcessService { * @param taskInstance * @return */ - public String formatTaskAppId(TaskInstance taskInstance){ + public String formatTaskAppId(TaskInstance taskInstance) { ProcessDefinition definition = this.findProcessDefineById(taskInstance.getProcessDefinitionId()); ProcessInstance processInstanceById = this.findProcessInstanceById(taskInstance.getProcessInstanceId()); - if(definition == null || processInstanceById == null){ + if (definition == null || processInstanceById == null) { return ""; } return String.format("%s_%s_%s", From 5b4a8fdcf594f8790188f215d0d7bb2d359c7d70 Mon Sep 17 00:00:00 2001 From: zhuangchong <37063904+zhuangchong@users.noreply.github.com> Date: Thu, 15 Oct 2020 19:52:23 +0800 Subject: [PATCH 38/43] [fix-3788][pom]spring framework Security breach (#3882) * spring framework Security breach,update springboot version. * update license. * update license. * update netty license. * update license. * Update known-dependencies.txt * Update known-dependencies.txt Co-authored-by: zhuangchong Co-authored-by: dailidong --- dolphinscheduler-dist/release-docs/LICENSE | 40 +++++----- dolphinscheduler-dist/release-docs/NOTICE | 4 +- pom.xml | 6 +- tools/dependencies/known-dependencies.txt | 92 +++++++++++----------- 4 files changed, 71 insertions(+), 71 deletions(-) diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 707ea5cab1..2bc80b300f 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -230,7 +230,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. clickhouse-jdbc 0.1.52: https://mvnrepository.com/artifact/ru.yandex.clickhouse/clickhouse-jdbc/0.1.52, Apache 2.0 commons-beanutils 1.7.0 https://mvnrepository.com/artifact/commons-beanutils/commons-beanutils/1.7.0, Apache 2.0 commons-cli 1.2: https://mvnrepository.com/artifact/commons-cli/commons-cli/1.2, Apache 2.0 - commons-codec 1.6: https://mvnrepository.com/artifact/commons-codec/commons-codec/1.6, Apache 2.0 + commons-codec 1.11: https://mvnrepository.com/artifact/commons-codec/commons-codec/1.11, Apache 2.0 commons-collections 3.2.2: https://mvnrepository.com/artifact/commons-collections/commons-collections/3.2.2, Apache 2.0 commons-collections4 4.1: https://mvnrepository.com/artifact/org.apache.commons/commons-collections4/4.1, Apache 2.0 commons-compress 1.4.1: https://mvnrepository.com/artifact/org.apache.commons/commons-compress/1.4.1, Apache 2.0 @@ -348,20 +348,20 @@ The text of each license is also included at licenses/LICENSE-[project].txt. snakeyaml 1.23: https://mvnrepository.com/artifact/org.yaml/snakeyaml/1.23, Apache 2.0 snappy 0.2: https://mvnrepository.com/artifact/org.iq80.snappy/snappy/0.2, Apache 2.0 snappy-java 1.0.4.1: https://github.com/xerial/snappy-java, Apache 2.0 - spring-aop 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-aop/5.1.5.RELEASE, Apache 2.0 - spring-beans 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-beans/5.1.5.RELEASE, Apache 2.0 - spring-boot 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot/2.1.3.RELEASE, Apache 2.0 - spring-boot-autoconfigure 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-autoconfigure/2.1.3.RELEASE, Apache 2.0 - spring-boot-starter 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter/2.1.3.RELEASE, Apache 2.0 - spring-boot-starter-aop 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-aop/2.1.3.RELEASE, Apache 2.0 - spring-boot-starter-jdbc 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jdbc/2.1.3.RELEASE, Apache 2.0 - spring-boot-starter-jetty 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jetty/2.1.3.RELEASE, Apache 2.0 - spring-boot-starter-json 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-json/2.1.3.RELEASE, Apache 2.0 - spring-boot-starter-logging 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-logging/2.1.3.RELEASE, Apache 2.0 - spring-boot-starter-web 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-web/2.1.3.RELEASE, Apache 2.0 - spring-context 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-context/5.1.5.RELEASE, Apache 2.0 - spring-core 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-core, Apache 2.0 - spring-expression 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-expression, Apache 2.0 + spring-aop 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-aop/5.1.18.RELEASE, Apache 2.0 + spring-beans 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-beans/5.1.18.RELEASE, Apache 2.0 + spring-boot 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot/2.1.17.RELEASE, Apache 2.0 + spring-boot-autoconfigure 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-autoconfigure/2.1.17.RELEASE, Apache 2.0 + spring-boot-starter 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter/2.1.17.RELEASE, Apache 2.0 + spring-boot-starter-aop 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-aop/2.1.17.RELEASE, Apache 2.0 + spring-boot-starter-jdbc 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jdbc/2.1.17.RELEASE, Apache 2.0 + spring-boot-starter-jetty 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jetty/2.1.17.RELEASE, Apache 2.0 + spring-boot-starter-json 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-json/2.1.17.RELEASE, Apache 2.0 + spring-boot-starter-logging 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-logging/2.1.17.RELEASE, Apache 2.0 + spring-boot-starter-web 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-web/2.1.17.RELEASE, Apache 2.0 + spring-context 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-context/5.1.18.RELEASE, Apache 2.0 + spring-core 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-core/5.1.18.RELEASE, Apache 2.0 + spring-expression 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-expression/5.1.18.RELEASE, Apache 2.0 springfox-core 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-core, Apache 2.0 springfox-schema 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-schema, Apache 2.0 springfox-spi 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-spi, Apache 2.0 @@ -369,13 +369,13 @@ The text of each license is also included at licenses/LICENSE-[project].txt. springfox-swagger2 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-swagger2/2.9.2, Apache 2.0 springfox-swagger-common 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-swagger-common/2.9.2, Apache 2.0 springfox-swagger-ui 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-swagger-ui/2.9.2, Apache 2.0 - spring-jcl 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-jcl/5.1.5.RELEASE, Apache 2.0 - spring-jdbc 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-jdbc/5.1.5.RELEASE, Apache 2.0 + spring-jcl 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-jcl/5.1.18.RELEASE, Apache 2.0 + spring-jdbc 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-jdbc/5.1.18.RELEASE, Apache 2.0 spring-plugin-core 1.2.0.RELEASE: https://mvnrepository.com/artifact/org.springframework.plugin/spring-plugin-core/1.2.0.RELEASE, Apache 2.0 spring-plugin-metadata 1.2.0.RELEASE: https://mvnrepository.com/artifact/org.springframework.plugin/spring-plugin-metadata/1.2.0.RELEASE, Apache 2.0 - spring-tx 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-tx/5.1.5.RELEASE, Apache 2.0 - spring-web 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-web/5.1.5.RELEASE, Apache 2.0 - spring-webmvc 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-webmvc/5.1.5.RELEASE, Apache 2.0 + spring-tx 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-tx/5.1.18.RELEASE, Apache 2.0 + spring-web 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-web/5.1.18.RELEASE, Apache 2.0 + spring-webmvc 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-webmvc/5.1.18.RELEASE, Apache 2.0 swagger-annotations 1.5.20: https://mvnrepository.com/artifact/io.swagger/swagger-annotations/1.5.20, Apache 2.0 swagger-bootstrap-ui 1.9.3: https://mvnrepository.com/artifact/com.github.xiaoymin/swagger-bootstrap-ui/1.9.3, Apache 2.0 swagger-models 1.5.20: https://mvnrepository.com/artifact/io.swagger/swagger-models/1.5.20, Apache 2.0 diff --git a/dolphinscheduler-dist/release-docs/NOTICE b/dolphinscheduler-dist/release-docs/NOTICE index 901659e689..fdc68efbf3 100644 --- a/dolphinscheduler-dist/release-docs/NOTICE +++ b/dolphinscheduler-dist/release-docs/NOTICE @@ -384,8 +384,8 @@ This product contains the Maven wrapper scripts from 'Maven Wrapper', that provi Spring Framework NOTICE ======================================================================== -Spring Framework 5.1.5.RELEASE -Copyright (c) 2002-2019 Pivotal, Inc. +Spring Framework 5.1.18.RELEASE +Copyright (c) 2002-2020 Pivotal, Inc. This product is licensed to you under the Apache License, Version 2.0 (the "License"). You may not use this product except in compliance with diff --git a/pom.xml b/pom.xml index 9e4934e833..f7609611f4 100644 --- a/pom.xml +++ b/pom.xml @@ -59,8 +59,8 @@ UTF-8 UTF-8 4.3.0 - 5.1.5.RELEASE - 2.1.3.RELEASE + 5.1.18.RELEASE + 2.1.17.RELEASE 1.8 1.2.3 2.7.3 @@ -71,7 +71,7 @@ 5.0.5 1.1.22 1.4.200 - 1.6 + 1.11 1.1.1 4.4.1 4.4.1 diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 4a7f1662f0..96089d5dcd 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -2,26 +2,26 @@ HikariCP-3.2.0.jar activation-1.1.jar ant-1.6.5.jar aopalliance-1.0.jar -apache-el-8.5.35.1.jar +apache-el-8.5.54.jar apacheds-i18n-2.0.0-M15.jar apacheds-kerberos-codec-2.0.0-M15.jar api-asn1-api-1.0.0-M20.jar api-util-1.0.0-M20.jar asm-3.1.jar -aspectjweaver-1.9.2.jar +aspectjweaver-1.9.6.jar audience-annotations-0.5.0.jar avro-1.7.4.jar aws-java-sdk-1.7.4.jar bonecp-0.8.0.RELEASE.jar -byte-buddy-1.9.10.jar +byte-buddy-1.9.16.jar classmate-1.4.0.jar clickhouse-jdbc-0.1.52.jar commons-cli-1.2.jar -commons-codec-1.6.jar +commons-codec-1.11.jar commons-collections-3.2.2.jar commons-collections4-4.1.jar commons-compress-1.4.1.jar -commons-compiler-3.0.12.jar +commons-compiler-3.0.16.jar commons-configuration-1.10.jar commons-daemon-1.0.13.jar commons-beanutils-1.7.0.jar @@ -44,7 +44,7 @@ datanucleus-core-4.1.6.jar datanucleus-rdbms-4.1.7.jar derby-10.14.2.0.jar druid-1.1.22.jar -gson-2.8.5.jar +gson-2.8.6.jar guava-20.0.jar guice-3.0.jar guice-servlet-3.0.jar @@ -65,7 +65,7 @@ hadoop-yarn-client-2.7.3.jar hadoop-yarn-common-2.7.3.jar hadoop-yarn-server-common-2.7.3.jar hamcrest-core-1.3.jar -hibernate-validator-6.0.14.Final.jar +hibernate-validator-6.0.20.Final.jar hive-common-2.1.0.jar hive-jdbc-2.1.0.jar hive-metastore-2.1.0.jar @@ -77,19 +77,19 @@ hive-storage-api-2.1.0.jar htrace-core-3.1.0-incubating.jar httpclient-4.4.1.jar httpcore-4.4.1.jar -httpmime-4.5.7.jar +httpmime-4.5.12.jar jackson-annotations-2.9.8.jar jackson-core-2.9.8.jar jackson-core-asl-1.9.13.jar jackson-databind-2.9.8.jar -jackson-datatype-jdk8-2.9.8.jar -jackson-datatype-jsr310-2.9.8.jar +jackson-datatype-jdk8-2.9.10.jar +jackson-datatype-jsr310-2.9.10.jar jackson-jaxrs-1.9.13.jar jackson-mapper-asl-1.9.13.jar -jackson-module-parameter-names-2.9.8.jar +jackson-module-parameter-names-2.9.10.jar jackson-xc-1.9.13.jar jamon-runtime-2.3.1.jar -janino-3.0.12.jar +janino-3.0.16.jar java-xmlbuilder-0.4.jar javax.activation-api-1.2.0.jar javax.annotation-api-1.3.2.jar @@ -100,7 +100,7 @@ javax.servlet-api-3.1.0.jar javolution-5.5.1.jar jaxb-api-2.3.1.jar jaxb-impl-2.2.3-1.jar -jboss-logging-3.3.2.Final.jar +jboss-logging-3.3.3.Final.jar jdo-api-3.0.1.jar jersey-client-1.9.jar jersey-core-1.9.jar @@ -110,21 +110,21 @@ jersey-server-1.9.jar jets3t-0.9.0.jar jettison-1.1.jar jetty-6.1.26.jar -jetty-continuation-9.4.14.v20181114.jar -jetty-http-9.4.14.v20181114.jar -jetty-io-9.4.14.v20181114.jar -jetty-security-9.4.14.v20181114.jar -jetty-server-9.4.14.v20181114.jar -jetty-servlet-9.4.14.v20181114.jar -jetty-servlets-9.4.14.v20181114.jar +jetty-continuation-9.4.31.v20200723.jar +jetty-http-9.4.31.v20200723.jar +jetty-io-9.4.31.v20200723.jar +jetty-security-9.4.31.v20200723.jar +jetty-server-9.4.31.v20200723.jar +jetty-servlet-9.4.31.v20200723.jar +jetty-servlets-9.4.31.v20200723.jar jetty-util-6.1.26.jar -jetty-util-9.4.14.v20181114.jar -jetty-webapp-9.4.14.v20181114.jar -jetty-xml-9.4.14.v20181114.jar +jetty-util-9.4.31.v20200723.jar +jetty-webapp-9.4.31.v20200723.jar +jetty-xml-9.4.31.v20200723.jar jline-0.9.94.jar jna-4.5.2.jar jna-platform-4.5.2.jar -joda-time-2.10.1.jar +joda-time-2.10.6.jar jpam-1.1.jar jsch-0.1.42.jar jsp-2.1-6.1.14.jar @@ -133,7 +133,7 @@ jsp-api-2.1.jar jsqlparser-2.1.jar jsr305-3.0.0.jar jta-1.1.jar -jul-to-slf4j-1.7.25.jar +jul-to-slf4j-1.7.30.jar junit-4.12.jar leveldbjni-all-1.8.jar libfb303-0.9.3.jar @@ -155,7 +155,7 @@ mybatis-plus-core-3.2.0.jar mybatis-plus-extension-3.2.0.jar mybatis-spring-2.0.2.jar netty-3.6.2.Final.jar -netty-all-4.1.33.Final.jar +netty-all-4.1.52.Final.jar opencsv-2.3.jar oshi-core-3.5.0.jar paranamer-2.3.jar @@ -169,27 +169,27 @@ slf4j-api-1.7.5.jar snakeyaml-1.23.jar snappy-0.2.jar snappy-java-1.0.4.1.jar -spring-aop-5.1.5.RELEASE.jar -spring-beans-5.1.5.RELEASE.jar -spring-boot-2.1.3.RELEASE.jar -spring-boot-autoconfigure-2.1.3.RELEASE.jar -spring-boot-starter-2.1.3.RELEASE.jar -spring-boot-starter-aop-2.1.3.RELEASE.jar -spring-boot-starter-jdbc-2.1.3.RELEASE.jar -spring-boot-starter-jetty-2.1.3.RELEASE.jar -spring-boot-starter-json-2.1.3.RELEASE.jar -spring-boot-starter-logging-2.1.3.RELEASE.jar -spring-boot-starter-web-2.1.3.RELEASE.jar -spring-context-5.1.5.RELEASE.jar -spring-core-5.1.5.RELEASE.jar -spring-expression-5.1.5.RELEASE.jar -spring-jcl-5.1.5.RELEASE.jar -spring-jdbc-5.1.5.RELEASE.jar +spring-aop-5.1.18.RELEASE.jar +spring-beans-5.1.18.RELEASE.jar +spring-boot-2.1.17.RELEASE.jar +spring-boot-autoconfigure-2.1.17.RELEASE.jar +spring-boot-starter-2.1.17.RELEASE.jar +spring-boot-starter-aop-2.1.17.RELEASE.jar +spring-boot-starter-jdbc-2.1.17.RELEASE.jar +spring-boot-starter-jetty-2.1.17.RELEASE.jar +spring-boot-starter-json-2.1.17.RELEASE.jar +spring-boot-starter-logging-2.1.17.RELEASE.jar +spring-boot-starter-web-2.1.17.RELEASE.jar +spring-context-5.1.18.RELEASE.jar +spring-core-5.1.18.RELEASE.jar +spring-expression-5.1.18.RELEASE.jar +spring-jcl-5.1.18.RELEASE.jar +spring-jdbc-5.1.18.RELEASE.jar spring-plugin-core-1.2.0.RELEASE.jar spring-plugin-metadata-1.2.0.RELEASE.jar -spring-tx-5.1.5.RELEASE.jar -spring-web-5.1.5.RELEASE.jar -spring-webmvc-5.1.5.RELEASE.jar +spring-tx-5.1.18.RELEASE.jar +spring-web-5.1.18.RELEASE.jar +spring-webmvc-5.1.18.RELEASE.jar springfox-core-2.9.2.jar springfox-schema-2.9.2.jar springfox-spi-2.9.2.jar @@ -210,4 +210,4 @@ xmlenc-0.52.jar xz-1.0.jar zookeeper-3.4.14.jar guava-retrying-2.0.0.jar -presto-jdbc-0.238.1.jar \ No newline at end of file +presto-jdbc-0.238.1.jar From 62fb2fd19b769c2fa4fa68a32675891104d3608c Mon Sep 17 00:00:00 2001 From: wangyangting Date: Thu, 15 Oct 2020 19:57:06 +0800 Subject: [PATCH 39/43] fix service comment (#3901) * fix service comment ## Before ```xml ... ``` ## After ```xml ... ``` * Update dolphinscheduler-binary.xml Co-authored-by: dailidong --- .../src/main/assembly/dolphinscheduler-binary.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml index 9b61c5e094..34f2cd2a9a 100644 --- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml +++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml @@ -112,8 +112,8 @@ - - + + ${basedir}/../dolphinscheduler-service/src/main/resources @@ -226,4 +226,4 @@ - \ No newline at end of file + From 29a083ca8f9a6def8a3a3dc405086190e5261587 Mon Sep 17 00:00:00 2001 From: dailidong Date: Fri, 16 Oct 2020 10:43:46 +0800 Subject: [PATCH 40/43] Update ThreadUtilsTest.java --- .../dolphinscheduler/common/threadutils/ThreadUtilsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java index 821399ed6c..53441f0463 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java @@ -37,7 +37,7 @@ public class ThreadUtilsTest { @Test public void testNewDaemonFixedThreadExecutor() { // create core size and max size are all 3 - ExecutorService testExec = ThreadUtils.newDaemonFixedThreadExecutor("test-exec-thread", 3); + ExecutorService testExec = ThreadUtils.newDaemonFixedThreadExecutor("test-exec-thread",3); for (int i = 0; i < 2; i++) { final int index = i; From ed3f08916f782f1451455914288ebbb2824f46e1 Mon Sep 17 00:00:00 2001 From: dailidong Date: Fri, 16 Oct 2020 13:16:03 +0800 Subject: [PATCH 41/43] Update ThreadUtilsTest.java --- .../common/threadutils/ThreadUtilsTest.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java index 53441f0463..3d44c553d3 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.common.threadutils; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors; import org.apache.dolphinscheduler.common.thread.ThreadUtils; + import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,13 +32,14 @@ import static org.junit.Assert.*; public class ThreadUtilsTest { private static final Logger logger = LoggerFactory.getLogger(ThreadUtilsTest.class); + /** * create a naming thread */ @Test public void testNewDaemonFixedThreadExecutor() { // create core size and max size are all 3 - ExecutorService testExec = ThreadUtils.newDaemonFixedThreadExecutor("test-exec-thread",3); + ExecutorService testExec = ThreadUtils.newDaemonFixedThreadExecutor("test-exec-thread", 3); for (int i = 0; i < 2; i++) { final int index = i; @@ -73,8 +75,8 @@ public class ThreadUtilsTest { // send heart beat work while (start.getTime().getTime() <= end.getTime().getTime()) { System.out.println("worker here"); - System.out.println(index ++); - start.add(Calendar.SECOND, 1); + System.out.println(index++); + start.add(Calendar.SECOND, 1); globalTimer.add(Calendar.SECOND, 1); } } @@ -102,19 +104,17 @@ public class ThreadUtilsTest { /** * test threadPoolExecutors with 3 workers and current each 5 tasks - * @throws InterruptedException */ @Test public void testThreadInfo() throws InterruptedException { ThreadPoolExecutors workers = ThreadPoolExecutors.getInstance("worker", 1); - for (int i = 0; i < 2; ++i ) { + for (int i = 0; i < 2; ++i) { int index = i; workers.execute(() -> { for (int j = 0; j < 1; ++j) { try { Thread.sleep(100); logger.info("worker %s is doing the task", index); -// workers.printStatus(); } catch (InterruptedException e) { logger.error("InterruptedException", e); } @@ -129,7 +129,7 @@ public class ThreadUtilsTest { * test a single daemon thread pool */ @Test - public void testNewDaemonSingleThreadExecutor() { + public void testNewDaemonSingleThreadExecutor() { ExecutorService threadTest = ThreadUtils.newDaemonSingleThreadExecutor("thread_test"); threadTest.execute(() -> { for (int i = 0; i < 2; ++i) { @@ -148,7 +148,7 @@ public class ThreadUtilsTest { ThreadPoolExecutor threadPoolExecutor = ThreadUtils.newDaemonCachedThreadPool("threadTest-"); Thread thread1 = threadPoolExecutor.getThreadFactory().newThread(() -> { for (int i = 0; i < 2; ++i) { - logger.info("this task is with index " + i ); + logger.info("this task is with index " + i); } }); assertTrue(thread1.getName().startsWith("threadTest-")); @@ -160,7 +160,7 @@ public class ThreadUtilsTest { @Test public void testNewDaemonCachedThreadPoolWithThreadNumber() { ThreadPoolExecutor threadPoolExecutor = ThreadUtils.newDaemonCachedThreadPool("threadTest--", 3, 10); - for (int i = 0; i < 2; ++ i) { + for (int i = 0; i < 2; ++i) { threadPoolExecutor.getThreadFactory().newThread(() -> { assertEquals(3, threadPoolExecutor.getActiveCount()); logger.info("this task is first work to do"); @@ -172,5 +172,4 @@ public class ThreadUtilsTest { } - } From 15436cfcdd7612011e3d02bc3c84da8958de16dc Mon Sep 17 00:00:00 2001 From: dailidong Date: Fri, 16 Oct 2020 13:49:57 +0800 Subject: [PATCH 42/43] Delete ThreadUtilsTest.java not useful test --- .../common/threadutils/ThreadUtilsTest.java | 175 ------------------ 1 file changed, 175 deletions(-) delete mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java deleted file mode 100644 index 3d44c553d3..0000000000 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.common.threadutils; - -import org.apache.dolphinscheduler.common.thread.Stopper; -import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors; -import org.apache.dolphinscheduler.common.thread.ThreadUtils; - -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Calendar; -import java.util.concurrent.*; - -import static org.junit.Assert.*; - - -public class ThreadUtilsTest { - private static final Logger logger = LoggerFactory.getLogger(ThreadUtilsTest.class); - - /** - * create a naming thread - */ - @Test - public void testNewDaemonFixedThreadExecutor() { - // create core size and max size are all 3 - ExecutorService testExec = ThreadUtils.newDaemonFixedThreadExecutor("test-exec-thread", 3); - - for (int i = 0; i < 2; i++) { - final int index = i; - testExec.submit(() -> { - logger.info("do some work index " + index); - }); - } - assertFalse(testExec.isShutdown()); - testExec.shutdownNow(); - assertTrue(testExec.isShutdown()); - - } - - /** - * test schedulerThreadExecutor as for print time in scheduler - * default check thread is 1 - */ - @Test - public void testNewDaemonScheduleThreadExecutor() { - - ScheduledExecutorService scheduleService = ThreadUtils.newDaemonThreadScheduledExecutor("scheduler-thread", 1); - Calendar start = Calendar.getInstance(); - Calendar globalTimer = Calendar.getInstance(); - globalTimer.set(2019, Calendar.DECEMBER, 1, 0, 0, 0); - // current - Calendar end = Calendar.getInstance(); - end.set(2019, Calendar.DECEMBER, 1, 0, 0, 3); - Runnable schedulerTask = new Runnable() { - @Override - public void run() { - start.set(2019, Calendar.DECEMBER, 1, 0, 0, 0); - int index = 0; - // send heart beat work - while (start.getTime().getTime() <= end.getTime().getTime()) { - System.out.println("worker here"); - System.out.println(index++); - start.add(Calendar.SECOND, 1); - globalTimer.add(Calendar.SECOND, 1); - } - } - }; - scheduleService.scheduleAtFixedRate(schedulerTask, 2, 10, TimeUnit.SECONDS); - assertFalse(scheduleService.isShutdown()); - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - scheduleService.shutdownNow(); - assertTrue(scheduleService.isShutdown()); - } - - /** - * test stopper is working normal - */ - @Test - public void testStopper() { - assertTrue(Stopper.isRunning()); - Stopper.stop(); - assertTrue(Stopper.isStopped()); - } - - /** - * test threadPoolExecutors with 3 workers and current each 5 tasks - */ - @Test - public void testThreadInfo() throws InterruptedException { - ThreadPoolExecutors workers = ThreadPoolExecutors.getInstance("worker", 1); - for (int i = 0; i < 2; ++i) { - int index = i; - workers.execute(() -> { - for (int j = 0; j < 1; ++j) { - try { - Thread.sleep(100); - logger.info("worker %s is doing the task", index); - } catch (InterruptedException e) { - logger.error("InterruptedException", e); - } - } - }); - } - Thread.sleep(1001); - workers.shutdown(); - } - - /** - * test a single daemon thread pool - */ - @Test - public void testNewDaemonSingleThreadExecutor() { - ExecutorService threadTest = ThreadUtils.newDaemonSingleThreadExecutor("thread_test"); - threadTest.execute(() -> { - for (int i = 0; i < 2; ++i) { - logger.info("daemon working "); - } - - }); - assertFalse(threadTest.isShutdown()); - threadTest.shutdownNow(); - assertTrue(threadTest.isShutdown()); - } - - @Test - public void testNewDaemonCachedThreadPool() { - - ThreadPoolExecutor threadPoolExecutor = ThreadUtils.newDaemonCachedThreadPool("threadTest-"); - Thread thread1 = threadPoolExecutor.getThreadFactory().newThread(() -> { - for (int i = 0; i < 2; ++i) { - logger.info("this task is with index " + i); - } - }); - assertTrue(thread1.getName().startsWith("threadTest-")); - assertFalse(threadPoolExecutor.isShutdown()); - threadPoolExecutor.shutdown(); - assertTrue(threadPoolExecutor.isShutdown()); - } - - @Test - public void testNewDaemonCachedThreadPoolWithThreadNumber() { - ThreadPoolExecutor threadPoolExecutor = ThreadUtils.newDaemonCachedThreadPool("threadTest--", 3, 10); - for (int i = 0; i < 2; ++i) { - threadPoolExecutor.getThreadFactory().newThread(() -> { - assertEquals(3, threadPoolExecutor.getActiveCount()); - logger.info("this task is first work to do"); - }); - } - assertFalse(threadPoolExecutor.isShutdown()); - threadPoolExecutor.shutdown(); - assertTrue(threadPoolExecutor.isShutdown()); - } - - -} From 2ee5df42b4c0310f465a1a9a0aafb8c426a05a3f Mon Sep 17 00:00:00 2001 From: dailidong Date: Fri, 16 Oct 2020 15:59:35 +0800 Subject: [PATCH 43/43] fix Zookeeper does not start, printed logs have no error messages, Api services can start but cannot be accessed --- .../service/zk/CuratorZookeeperClient.java | 19 +++++++++++++------ .../zk/CuratorZookeeperClientTest.java | 3 +-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClient.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClient.java index c08da0ef72..5a04c5a23b 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClient.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClient.java @@ -24,6 +24,7 @@ import org.apache.curator.framework.state.ConnectionState; import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.data.ACL; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; @@ -32,6 +33,7 @@ import org.springframework.stereotype.Component; import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.concurrent.TimeUnit; import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull; @@ -55,9 +57,10 @@ public class CuratorZookeeperClient implements InitializingBean { } private CuratorFramework buildClient() { - logger.info("zookeeper registry center init, server lists is: {}.", zookeeperConfig.getServerList()); + logger.info("zookeeper registry center init, server lists is: [{}]", zookeeperConfig.getServerList()); - CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder().ensembleProvider(new DefaultEnsembleProvider(checkNotNull(zookeeperConfig.getServerList(),"zookeeper quorum can't be null"))) + CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder() + .ensembleProvider(new DefaultEnsembleProvider(checkNotNull(zookeeperConfig.getServerList(), "zookeeper quorum can't be null"))) .retryPolicy(new ExponentialBackoffRetry(zookeeperConfig.getBaseSleepTimeMs(), zookeeperConfig.getMaxRetries(), zookeeperConfig.getMaxSleepMs())); //these has default value @@ -84,7 +87,9 @@ public class CuratorZookeeperClient implements InitializingBean { zkClient = builder.build(); zkClient.start(); try { - zkClient.blockUntilConnected(); + logger.info("trying to connect zookeeper server list:{}", zookeeperConfig.getServerList()); + zkClient.blockUntilConnected(30, TimeUnit.SECONDS); + } catch (final Exception ex) { throw new RuntimeException(ex); } @@ -95,12 +100,14 @@ public class CuratorZookeeperClient implements InitializingBean { checkNotNull(zkClient); zkClient.getConnectionStateListenable().addListener((client, newState) -> { - if(newState == ConnectionState.LOST){ + if (newState == ConnectionState.LOST) { logger.error("connection lost from zookeeper"); - } else if(newState == ConnectionState.RECONNECTED){ + } else if (newState == ConnectionState.RECONNECTED) { logger.info("reconnected to zookeeper"); - } else if(newState == ConnectionState.SUSPENDED){ + } else if (newState == ConnectionState.SUSPENDED) { logger.warn("connection SUSPENDED to zookeeper"); + } else if (newState == ConnectionState.CONNECTED) { + logger.info("connected to zookeeper server list:[{}]", zookeeperConfig.getServerList()); } }); } diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClientTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClientTest.java index c0297799ea..b1c2ec5e25 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClientTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClientTest.java @@ -59,9 +59,8 @@ public class CuratorZookeeperClientTest { zookeeperConfig.setDsRoot("/dolphinscheduler"); zookeeperConfig.setMaxWaitTime(30000); zookeeperClient.setZookeeperConfig(zookeeperConfig); - System.out.println("start"); zookeeperClient.afterPropertiesSet(); - System.out.println("end"); + Assert.assertNotNull(zookeeperClient.getZkClient()); } } \ No newline at end of file